From fc1d0f5af3127d68c863ef9c4b9ffe6663e5e77b Mon Sep 17 00:00:00 2001 From: Ivor Bosloper Date: Sat, 12 Sep 2026 12:24:32 +0200 Subject: [PATCH] JP: four editions, and a date per parcel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Fude polygons are published per year and the converter knew one of them. All four are variants now, newest first, with the test fixture last so it is not chosen by accident. The determination date used to be a constant for the whole file. Each parcel carries its own issue year, which becomes its determination:datetime in the SQL migration — the point of converting this dataset with SQL rather than in memory: 33 GB of Parquet across the four editions. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01DVx9uQV2QPM8ecPAY3ZjXG --- CHANGELOG.md | 1 + fiboa_cli/datasets/jp.py | 19 +++++-------------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2121e4b8..0e7f9e8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - ES-CL: the ITACyL server is https-only, the 2025 shapefiles sit in province subfolders, and C_REFREC is the identifier - A test refuses a fixture above 5 MB, committed or merely lying in the fixture folder, because a failing convert test downloads the real source there - ES-MD: find RECINTO.shp wherever the archive puts it +- JP: editions 2021-2024, each with the determination date of the parcel rather than a constant - PerFileBaseConverter: convert a multi-file source one file at a time and merge the parts, so a dataset larger than memory can be converted; used by the Spain-wide converter - FiboaDuckDBBaseConverter: convert a source that is already Parquet with SQL, without loading it into memory - Update vecorel-cli to v0.2.17: diff --git a/fiboa_cli/datasets/jp.py b/fiboa_cli/datasets/jp.py index 57c31a24..aed27487 100644 --- a/fiboa_cli/datasets/jp.py +++ b/fiboa_cli/datasets/jp.py @@ -1,14 +1,13 @@ -import pandas as pd +from fiboa_cli.conversion.duckdb import FiboaDuckDBBaseConverter -from ..conversion.fiboa_converter import FiboaBaseConverter - -class JPConverter(FiboaBaseConverter): +class JPConverter(FiboaDuckDBBaseConverter): variants = { "2024": "https://data.source.coop/pacificspatial/field-polygon-jp/parquet/jp_field_polygons_2024.parquet", "2023": "https://data.source.coop/pacificspatial/field-polygon-jp/parquet/jp_field_polygons_2023.parquet", "2022": "https://data.source.coop/pacificspatial/field-polygon-jp/parquet/jp_field_polygons_2022.parquet", "2021": "https://data.source.coop/pacificspatial/field-polygon-jp/parquet/jp_field_polygons_2021.parquet", + "test": "./tests/data-files/convert/jp/jp_field_polygons_2024.parquet", } id = "jp" @@ -32,21 +31,13 @@ class JPConverter(FiboaBaseConverter): "local_government_cd": "admin_local_code", "issue_year": "determination:datetime", } + # SQL migrations (DuckDB converter): per-feature determination date from the issue year column_migrations = { - "issue_year": lambda col: pd.to_datetime(col, format="%Y"), + "issue_year": "make_timestamp(CAST(issue_year AS INTEGER), 1, 1, 0, 0, 0) AT TIME ZONE 'UTC'", } - missing_schemas = { "properties": { "land_type_en": {"type": "string"}, "admin_local_code": {"type": "string"}, } } - - def convert(self, *args, **kwargs): - # Open only these columns to limit memory usage - super().convert( - *args, - columns=["GEOM", "polygon_uuid", "land_type_en", "local_government_cd", "issue_year"], - **kwargs, - )