Implement DuckDB-based converter

CHANGELOG.md

@@ -38,6 +38,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
   - Collection-only properties are kept when merging collections
   - Default GeoParquet compression is now zstd (level 15), configurable via `--compression_level`
 - DE-SH: make the 2023, 2025 and 2026 editions convert — glob the GeoPackage inside the archive (2023 was written with user_version = 0, so the archive alone matches no driver), parse fachguelti as DD.MM.YYYY, and map the 2023 and upper-case 2025/2026 column spellings that silently dropped determination:datetime and metrics:area (their area is text with a decimal comma)
+- Add DuckDB BaseConverter for efficiently transforming large datasets
+- Update vecorel-cli to v0.2.18:
+  - The DuckDB converter base and the GeoParquet post-processing live there now
+  - Converters record the collection id in the collection metadata instead of a constant `collection` column
+  - Converters drop rows that can never validate (missing required values, empty or missing geometries), bounded by `max_dropped_share`
+  - Converters fail when both `sources` and `variants` are declared (fixed in the HR converter), and warn when no column is mapped to `id` or the id column is not unique
+  - Converters load all schemas upfront with retries
 
 ## [v0.21.0] - 2026-02-16
 
@@ -49,6 +56,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Fix the column additions of the determination fields in the AI4SF converter
 - Add HCAT to datasets where possible
 - Updated years & variants for at_crop, be_vlg, es_an, es_cl, es_pv, ie, pt, se
+- Add DuckDB BaseConverter for efficiently transforming large datasets
 - Extend create_stac, include include fiboa data
 - Publish command; skip hidden files, generate better texts
 - Fix to vecorel: converter.license and provider should be string

fiboa_cli/conversion/duckdb.py

@@ -0,0 +1,10 @@
+from vecorel_cli.conversion.duckdb import DuckDBBaseConverter
+
+from .fiboa_converter import FiboaBaseConverter
+
+
+# This converter is experimental, use with caution.
+# Results may not be fully fiboa compliant yet.
+# Use this primarily for datasets that are too large to be processed by the default converter
+class FiboaDuckDBBaseConverter(DuckDBBaseConverter, FiboaBaseConverter):
+    pass

fiboa_cli/datasets/commons/euro_land.py

@@ -47,3 +47,11 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         provider = "Europe-LAND HE Project <https://doi.org/10.5281/zenodo.14230620>"
         self.provider = (f"{self.provider}, {provider}") if self.provider else provider
+
+    def migrate(self, gdf):
+        # Some Europe-LAND files (e.g. LT 2024) ship an empty crop_code column next to
+        # a populated crop_name; the name is then the best available crop code.
+        if "crop_code" in gdf.columns and gdf["crop_code"].isna().all():
+            self.warning("crop_code is empty, using crop_name as crop:code")
+            gdf["crop_code"] = gdf["crop_name"]
+        return super().migrate(gdf)

fiboa_cli/datasets/hr.py

@@ -7,7 +7,6 @@
 
 
 class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter):
-    sources = "https://www.apprrr.hr/wp-content/uploads/nipp/land_parcels.gpkg"
     variants = {
         "2024": f"{base}/land_parcels.gpkg",
         **{str(y): f"{base}/arkod_31_12_{y}.gpkg" for y in range(2023, 2010, -1)},

fiboa_cli/datasets/jp.py

@@ -1,10 +1,9 @@
-import pandas as pd
+from fiboa_cli.conversion.duckdb import FiboaDuckDBBaseConverter
 
-from ..conversion.fiboa_converter import FiboaBaseConverter
 
-
-class JPConverter(FiboaBaseConverter):
+class JPConverter(FiboaDuckDBBaseConverter):
     variants = {
+        "test": "./tests/data-files/convert/jp/jp_field_polygons_2024.parquet",
         "2024": "https://data.source.coop/pacificspatial/field-polygon-jp/parquet/jp_field_polygons_2024.parquet",
         "2023": "https://data.source.coop/pacificspatial/field-polygon-jp/parquet/jp_field_polygons_2023.parquet",
         "2022": "https://data.source.coop/pacificspatial/field-polygon-jp/parquet/jp_field_polygons_2022.parquet",
@@ -30,23 +29,11 @@ class JPConverter(FiboaBaseConverter):
         "polygon_uuid": "id",
         "land_type_en": "land_type_en",
         "local_government_cd": "admin_local_code",
-        "issue_year": "determination:datetime",
-    }
-    column_migrations = {
-        "issue_year": lambda col: pd.to_datetime(col, format="%Y"),
     }
-
+    column_additions = {"determination:datetime": "2024-01-01T00:00:00Z"}
     missing_schemas = {
         "properties": {
             "land_type_en": {"type": "string"},
             "admin_local_code": {"type": "string"},
         }
     }
-
-    def convert(self, *args, **kwargs):
-        # Open only these columns to limit memory usage
-        super().convert(
-            *args,
-            columns=["GEOM", "polygon_uuid", "land_type_en", "local_government_cd", "issue_year"],
-            **kwargs,
-        )

fiboa_cli/datasets/lt.py

@@ -1,5 +1,7 @@
 from fiboa_cli.datasets.commons.euro_land import EuroLandBaseConverter
 
+BASE = "https://zenodo.org/records/18670815/files"
+
 
 class LTConverter(EuroLandBaseConverter):
     id = "lt"
@@ -10,4 +12,12 @@ class LTConverter(EuroLandBaseConverter):
     provider = "Nacionalinė mokėjimo agentūra prie Žemės ūkio ministerijos <https://www.nma.lt>"
     attribution = "Nacionalinė mokėjimo agentūra prie Žemės ūkio ministerijos"
     ec_mapping_csv = "lt_2021.csv"
-    sources = {"https://zenodo.org/records/14384070/files/LT_2024.zip": ["GSA-LT-2024.geoparquet"]}
+    # Europe-LAND v1.3 (record 18670815, February 2026) bundles both editions in
+    # one archive; v1.1 (record 14384070), which this converter used to read,
+    # carried 2024 alone.
+    variants = {
+        str(year): {f"{BASE}/LT_years_2024-2025.zip": [f"GSA-LT-{year}.geoparquet"]}
+        for year in (2025, 2024)
+    }
+    # The inventory carries no per-feature date, so the edition is the year.
+    use_variant_as_determination = True