From da8101000cffb6a42897a5bfd3bdb4cf03416d50 Mon Sep 17 00:00:00 2001 From: Ivor Bosloper Date: Fri, 11 Sep 2026 21:25:35 +0200 Subject: [PATCH 1/2] BE-VLG: campaigns 2018-2026, and an id that identifies a field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Flemish portal keeps every campaign from 2018; the catalog published 2023 onwards. Adding the older ones turned up three things, two of which were true of the editions already published: - **REF_ID references the parcel, not the field.** A row is one crop declared on it, so it repeats whenever a parcel carries more than one: 183 references over 367 of the 515,747 rows of 2018, one of them winter barley and winter wheat on the same 0.6238 ha, another the same crop twice. It is published as `block_id` now and the row index identifies the field, which is safe because an edition is one layer of one file. 2023 repeated it 15 times and 2026 1,800 times, so this is not only about the old campaigns. - **The 2026 GeoPackage carries a QGIS `layer_styles` table** beside the parcels, and its single row was read as a field. It never reached the published file, but only because the empty-geometry guard dropped it at the end — luck, not filtering. - **The 2020 archive** holds a GeoPackage whose name is not the archive's (`Landbouwgebruikspercelen2020_igb15-05_uitgebreid_(toestand_19-03-2021).gpkg` inside `..._2020_uitgebreid_toestand_19-03-2021_GPKG.zip`), and its text is cp1252, not UTF-8: a byte no UTF-8 decoder accepts kills the read before the first row. Every other value in that file is ASCII, so cp1252 costs nothing, and it is set for that campaign only. - **The 2020 GeoPackage also declares "Undefined geographic SRS"** while holding Lambert 72 metres, which put 258615.94 in the STAC extent as an east longitude and built the Hilbert grid from a global extent. The CRS is set where the source leaves it out. All nine editions are built and published: 515,747 fields in 2018 to 597,088 in 2026, 5,109,729 field-years, unique ids throughout. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01DVx9uQV2QPM8ecPAY3ZjXG --- CHANGELOG.md | 1 + fiboa_cli/datasets/be_vlg.py | 63 ++++++++++++++++++++++++++++++++---- 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a654d7dd..6ccfafdb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - ES-CL: the ITACyL server is https-only, the 2025 shapefiles sit in province subfolders, and C_REFREC is the identifier - A test refuses a fixture above 5 MB, committed or merely lying in the fixture folder, because a failing convert test downloads the real source there - ES-MD: find RECINTO.shp wherever the archive puts it +- BE-VLG: editions 2018-2026; REF_ID is published as block_id and the row index identifies the field, the QGIS styles table in the 2026 GeoPackage is skipped, and the 2020 archive is read as cp1252 with its CRS declared - Update vecorel-cli to v0.2.17: - GeoJSON is read as UTF-8 as the format mandates, instead of the platform locale (cp1252 on Windows mangled umlauts) - GeoJSON files with a byte order mark no longer fail to read diff --git a/fiboa_cli/datasets/be_vlg.py b/fiboa_cli/datasets/be_vlg.py index 4d4c253a..4154576d 100644 --- a/fiboa_cli/datasets/be_vlg.py +++ b/fiboa_cli/datasets/be_vlg.py @@ -7,9 +7,12 @@ class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): + # Each archive holds one GeoPackage, but not under the name of the zip + # (2020 least of all), so glob for it rather than deriving it. variants = { - str(k): {PREFIX + v: [v.replace("_GPKG.zip", ".gpkg")]} + str(k): {PREFIX + v: ["*.gpkg"]} for k, v in ( + (2026, "agpa_2026_2026-06-02_public.zip"), (2025, "Landbouwgebruikspercelen_2025_-_Voorlopig_(extractie_02-06-2025)_GPKG.zip"), (2024, "Landbouwgebruikspercelen_2024_-_Definitief_(extractie_27-03-2025)_GPKG.zip"), (2023, "Landbouwgebruikspercelen_2023_-_Definitief_(extractie_28-03-2024)_GPKG.zip"), @@ -34,17 +37,65 @@ class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): attribution = "Bron: Dept. LV" license = "Licentie modellicentie-gratis-hergebruik/v1.0 " + # The 2020 GeoPackage holds a byte no UTF-8 decoder accepts, and everything + # else in it is ASCII. Only that campaign: forcing cp1252 on the others + # would mangle an accented crop name instead of failing. + CP1252_EDITIONS = {"2020"} + + def layer_filter(self, layer, uri): + # The 2026 GeoPackage carries a QGIS "layer_styles" table whose single + # row was read as a field. + return layer != "layer_styles" + + def read_data(self, paths, **kwargs): + if self.variant in self.CP1252_EDITIONS: + kwargs["encoding"] = "cp1252" + return super().read_data(paths, **kwargs) + + # the 2026 "agpa" edition renamed every column to English + RENAMES_2026 = { + "reference_id": "REF_ID", + "maincrop_code": "GWSCOD_H", + "maincrop_title": "GWSNAM_H", + "area_ha": "GRAF_OPP", + } + + # Flanders publishes in Lambert 72; the 2020 snapshot leaves its SRS + # undefined while holding those metres, which put 258615.94 into the STAC + # extent as an east longitude. + LAMBERT_72 = "EPSG:31370" + + def migrate(self, gdf): + if gdf.crs is None or gdf.crs.to_epsg() is None: + gdf = gdf.set_crs(self.LAMBERT_72, allow_override=True) + if "maincrop_code" in gdf.columns: + gdf = gdf.rename(columns=self.RENAMES_2026) + if "BT_OMSCH" not in gdf.columns: # no farm-typology column any more + gdf["BT_OMSCH"] = None + return super().migrate(gdf) + + # REF_ID references the parcel and a row is one crop declared on it, so it + # repeats: 183 references cover 367 of the 515,747 rows of 2018. It is + # published as block_id; the row index identifies the field, which is safe + # because an edition is one layer of one file. + index_as_id = True columns = { "geometry": "geometry", + "id": "id", "BT_OMSCH": "typology", "GRAF_OPP": "metrics:area", - "REF_ID": "id", + "REF_ID": "block_id", "GWSCOD_H": "crop:code", "GWSNAM_H": "crop:name", } - column_additions = { - "determination:datetime": "2024-03-28T00:00:00Z", - } + # Each edition is the campaign year of its variant; the old constant + # "2024-03-28" was the extraction date of one edition applied to all of them. + use_variant_as_determination = True ec_mapping_csv = "be_vlg_2021.csv" - missing_schemas = {"properties": {"typology": {"type": "string"}}} + missing_schemas = { + "properties": { + "typology": {"type": "string"}, + "block_id": {"type": "string"}, + } + } From 397e3cb242011a475d0085488553d9b7fcd7af29 Mon Sep 17 00:00:00 2001 From: Ivor Date: Mon, 14 Sep 2026 14:24:02 +0200 Subject: [PATCH 2/2] Simplify --- fiboa_cli/datasets/be_vlg.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/fiboa_cli/datasets/be_vlg.py b/fiboa_cli/datasets/be_vlg.py index 4154576d..e5aab6c5 100644 --- a/fiboa_cli/datasets/be_vlg.py +++ b/fiboa_cli/datasets/be_vlg.py @@ -37,9 +37,7 @@ class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): attribution = "Bron: Dept. LV" license = "Licentie modellicentie-gratis-hergebruik/v1.0 " - # The 2020 GeoPackage holds a byte no UTF-8 decoder accepts, and everything - # else in it is ASCII. Only that campaign: forcing cp1252 on the others - # would mangle an accented crop name instead of failing. + # Fix cp1252 encoding for 2020 CP1252_EDITIONS = {"2020"} def layer_filter(self, layer, uri): @@ -60,9 +58,6 @@ def read_data(self, paths, **kwargs): "area_ha": "GRAF_OPP", } - # Flanders publishes in Lambert 72; the 2020 snapshot leaves its SRS - # undefined while holding those metres, which put 258615.94 into the STAC - # extent as an east longitude. LAMBERT_72 = "EPSG:31370" def migrate(self, gdf): @@ -74,10 +69,6 @@ def migrate(self, gdf): gdf["BT_OMSCH"] = None return super().migrate(gdf) - # REF_ID references the parcel and a row is one crop declared on it, so it - # repeats: 183 references cover 367 of the 515,747 rows of 2018. It is - # published as block_id; the row index identifies the field, which is safe - # because an edition is one layer of one file. index_as_id = True columns = { "geometry": "geometry", @@ -88,8 +79,6 @@ def migrate(self, gdf): "GWSCOD_H": "crop:code", "GWSNAM_H": "crop:name", } - # Each edition is the campaign year of its variant; the old constant - # "2024-03-28" was the extraction date of one edition applied to all of them. use_variant_as_determination = True ec_mapping_csv = "be_vlg_2021.csv"