diff --git a/CHANGELOG.md b/CHANGELOG.md index 2121e4b8..0e7f9e8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - ES-CL: the ITACyL server is https-only, the 2025 shapefiles sit in province subfolders, and C_REFREC is the identifier - A test refuses a fixture above 5 MB, committed or merely lying in the fixture folder, because a failing convert test downloads the real source there - ES-MD: find RECINTO.shp wherever the archive puts it +- JP: editions 2021-2024, each with the determination date of the parcel rather than a constant - PerFileBaseConverter: convert a multi-file source one file at a time and merge the parts, so a dataset larger than memory can be converted; used by the Spain-wide converter - FiboaDuckDBBaseConverter: convert a source that is already Parquet with SQL, without loading it into memory - Update vecorel-cli to v0.2.17: diff --git a/fiboa_cli/datasets/jp.py b/fiboa_cli/datasets/jp.py index 57c31a24..aed27487 100644 --- a/fiboa_cli/datasets/jp.py +++ b/fiboa_cli/datasets/jp.py @@ -1,14 +1,13 @@ -import pandas as pd +from fiboa_cli.conversion.duckdb import FiboaDuckDBBaseConverter -from ..conversion.fiboa_converter import FiboaBaseConverter - -class JPConverter(FiboaBaseConverter): +class JPConverter(FiboaDuckDBBaseConverter): variants = { "2024": "https://data.source.coop/pacificspatial/field-polygon-jp/parquet/jp_field_polygons_2024.parquet", "2023": "https://data.source.coop/pacificspatial/field-polygon-jp/parquet/jp_field_polygons_2023.parquet", "2022": "https://data.source.coop/pacificspatial/field-polygon-jp/parquet/jp_field_polygons_2022.parquet", "2021": "https://data.source.coop/pacificspatial/field-polygon-jp/parquet/jp_field_polygons_2021.parquet", + "test": "./tests/data-files/convert/jp/jp_field_polygons_2024.parquet", } id = "jp" @@ -32,21 +31,13 @@ class JPConverter(FiboaBaseConverter): "local_government_cd": "admin_local_code", "issue_year": "determination:datetime", } + # SQL migrations (DuckDB converter): per-feature determination date from the issue year column_migrations = { - "issue_year": lambda col: pd.to_datetime(col, format="%Y"), + "issue_year": "make_timestamp(CAST(issue_year AS INTEGER), 1, 1, 0, 0, 0) AT TIME ZONE 'UTC'", } - missing_schemas = { "properties": { "land_type_en": {"type": "string"}, "admin_local_code": {"type": "string"}, } } - - def convert(self, *args, **kwargs): - # Open only these columns to limit memory usage - super().convert( - *args, - columns=["GEOM", "polygon_uuid", "land_type_en", "local_government_cd", "issue_year"], - **kwargs, - )