From adaa61aa98a8a0ab7ee0ac9a5816292c3bd37be7 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Mon, 10 Aug 2026 17:26:46 +0200 Subject: [PATCH 01/11] add tests, update doc, core logic Signed-off-by: Konstantin Morozov --- docs/en/antalya/part_export.md | 15 +- docs/en/antalya/partition_export.md | 15 +- src/Core/Settings.cpp | 5 +- src/Core/SettingsEnums.h | 1 + src/Storages/MergeTree/ExportPartTask.cpp | 20 +-- .../MergeTree/ExportPartitionUtils.cpp | 123 ++++++++++++--- src/Storages/MergeTree/ExportPartitionUtils.h | 14 +- .../tests/gtest_export_partition_ordering.cpp | 110 +++++++++++++ .../test.py | 144 +++++++++++++++--- .../test.py | 55 ++++++- .../test.py | 61 +++++--- .../test.py | 71 ++++++++- 12 files changed, 542 insertions(+), 92 deletions(-) diff --git a/docs/en/antalya/part_export.md b/docs/en/antalya/part_export.md index ef7a7e25f5f7..fc4cefdafb51 100644 --- a/docs/en/antalya/part_export.md +++ b/docs/en/antalya/part_export.md @@ -140,11 +140,20 @@ In case a table function is used as the destination, the schema can be omitted a - **Type**: `MergeTreePartExportSchemaMismatchMode` - **Default**: `strict` -- **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` allows a column-count mismatch between the source `MergeTree` table and the destination table. Columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. Possible values: - - `strict` - the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. +- **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` allows the source `MergeTree` table to have more columns than the destination table and how destination columns are matched in that case. Possible values: + - `strict` - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`, and the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - `ignore_extra_source_columns_by_position` - the source may have more columns than the destination. The extra trailing source columns (by position) are dropped and not exported. The destination having more columns than the source is still rejected in this mode. + - `ignore_extra_source_columns_by_name` - the source may have more columns than the destination. Every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be reordered and additional source columns may occur in any position. Unmatched source columns are not exported. If a destination column is absent from the source, including when it was renamed, the export throws `THERE_IS_NO_COLUMN`; there is no positional fallback. Use `ignore_extra_source_columns_by_position` when renamed columns should be matched by position. - The extra trailing source columns are still read and evaluated (including `MATERIALIZED`/`ALIAS` columns, and any column another kept column's `ALIAS`/`MATERIALIZED` expression depends on) before being dropped, so this setting only changes which columns end up in the destination, not what is computed while reading the part. + Extra source columns are still read and evaluated (including `MATERIALIZED`/`ALIAS` columns, and any column another kept column's `ALIAS`/`MATERIALIZED` expression depends on) before being dropped, so this setting only changes which columns end up in the destination, not what is computed while reading the part. Type conversion and `export_merge_tree_part_allow_lossy_cast` are applied after columns are matched. + + Error behavior: + + - In `strict` mode, a different number of source and destination columns throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. + - In `ignore_extra_source_columns_by_position` mode, the destination having more columns than the source throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. + - In `ignore_extra_source_columns_by_name` mode, a destination column which is absent from the source throws `THERE_IS_NO_COLUMN`. + - Renaming a destination column in `ignore_extra_source_columns_by_name` mode throws `THERE_IS_NO_COLUMN`; the mode does not fall back to positional matching. + - After columns have been matched successfully, a cast rejected by the export type-safety check throws `INCOMPATIBLE_COLUMNS`. ## Examples diff --git a/docs/en/antalya/partition_export.md b/docs/en/antalya/partition_export.md index a3b9e3dfc4f2..a57f38ba6ac9 100644 --- a/docs/en/antalya/partition_export.md +++ b/docs/en/antalya/partition_export.md @@ -132,11 +132,20 @@ Notes: - **Type**: `MergeTreePartExportSchemaMismatchMode` - **Default**: `strict` -- **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` allows a column-count mismatch between the source `MergeTree` table and the destination table. Columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. Possible values: - - `strict` - the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. +- **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` allows the source `MergeTree` table to have more columns than the destination table and how destination columns are matched in that case. Possible values: + - `strict` - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`, and the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - `ignore_extra_source_columns_by_position` - the source may have more columns than the destination. The extra trailing source columns (by position) are dropped and not exported. The destination having more columns than the source is still rejected in this mode. + - `ignore_extra_source_columns_by_name` - the source may have more columns than the destination. Every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be reordered and additional source columns may occur in any position. Unmatched source columns are not exported. If a destination column is absent from the source, including when it was renamed, the export throws `THERE_IS_NO_COLUMN`; there is no positional fallback. Use `ignore_extra_source_columns_by_position` when renamed columns should be matched by position. - The extra trailing source columns are still read and evaluated (including `MATERIALIZED`/`ALIAS` columns, and any column another kept column's `ALIAS`/`MATERIALIZED` expression depends on) before being dropped, so this setting only changes which columns end up in the destination, not what is computed while reading the part. + Extra source columns are still read and evaluated (including `MATERIALIZED`/`ALIAS` columns, and any column another kept column's `ALIAS`/`MATERIALIZED` expression depends on) before being dropped, so this setting only changes which columns end up in the destination, not what is computed while reading the part. Type conversion and `export_merge_tree_part_allow_lossy_cast` are applied after columns are matched. + + Error behavior: + + - In `strict` mode, a different number of source and destination columns throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. + - In `ignore_extra_source_columns_by_position` mode, the destination having more columns than the source throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. + - In `ignore_extra_source_columns_by_name` mode, a destination column which is absent from the source throws `THERE_IS_NO_COLUMN`. + - Renaming a destination column in `ignore_extra_source_columns_by_name` mode throws `THERE_IS_NO_COLUMN`; the mode does not fall back to positional matching. + - After columns have been matched successfully, a cast rejected by the export type-safety check throws `INCOMPATIBLE_COLUMNS`. ## Examples diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 4109ce49ac12..52ce73c6bf24 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -7628,10 +7628,11 @@ Allow `EXPORT PART`/`EXPORT PARTITION` to apply lossy (non-value-preserving) cas When exporting to Apache Iceberg, the partition value written to the metadata is derived from the source partition columns by casting them to the destination partition-field types and applying the destination partition transform — the same computation the exported data files use, so the metadata stays consistent with the data. A lossy cast on a partition column remains semantically truncating: both the data files and the metadata contain the truncated value, and such casts require this setting to be enabled. )", 0) \ DECLARE(MergeTreePartExportSchemaMismatchMode, export_merge_tree_part_schema_mismatch_mode, MergeTreePartExportSchemaMismatchMode::strict, R"( -Controls whether `EXPORT PART`/`EXPORT PARTITION` allows a column-count mismatch between the source `MergeTree` table and the destination table. Columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. +Controls whether `EXPORT PART`/`EXPORT PARTITION` allows the source `MergeTree` table to have more columns than the destination table and how destination columns are matched. Possible values: -- `strict` (default) - the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. +- `strict` (default) - columns are matched positionally and the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - `ignore_extra_source_columns_by_position` - the source may have more columns than the destination. The extra trailing source columns (by position) are dropped and not exported. The destination having more columns than the source is still rejected in this mode. +- `ignore_extra_source_columns_by_name` - every destination column is matched to a source column with the same exact, case-sensitive name. Destination columns may be reordered and additional source columns may occur in any position. A destination column absent from the source throws `THERE_IS_NO_COLUMN`; there is no positional fallback. )", 0) \ \ /* ####################################################### */ \ diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index 0c82d40345cd..f5c48509e0b0 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -578,6 +578,7 @@ enum class MergeTreePartExportSchemaMismatchMode : uint8_t { strict, ignore_extra_source_columns_by_position, + ignore_extra_source_columns_by_name, }; DECLARE_SETTING_ENUM(MergeTreePartExportSchemaMismatchMode) diff --git a/src/Storages/MergeTree/ExportPartTask.cpp b/src/Storages/MergeTree/ExportPartTask.cpp index 5ef537bbef1c..8aa3511afad4 100644 --- a/src/Storages/MergeTree/ExportPartTask.cpp +++ b/src/Storages/MergeTree/ExportPartTask.cpp @@ -113,15 +113,6 @@ namespace } } - /// Mirrors `InterpreterInsertQuery::addInsertToSelectPipeline`: positional match, - /// destination header = `getSampleBlockNonMaterialized()`, all type bridging is done - /// by the CAST inside `makeConvertingActions`. No pre-validation, no per-column - /// lossy/non-lossy classification — restrictions are exactly what INSERT SELECT enforces. - /// - /// Exception: when `export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_position'` - /// and the source has more columns than the destination, the extra trailing source - /// columns (by position) are dropped by a preliminary projection step before the - /// positional convert, so `makeConvertingActions` always sees equal-sized inputs. void addExportConvertingActions( QueryPlan & plan_for_part, const IStorage & destination_storage, @@ -131,9 +122,10 @@ namespace = destination_storage.getInMemoryMetadataPtr()->getSampleBlockNonMaterialized(); const auto & destination_columns = destination_header.getColumnsWithTypeAndName(); + const auto schema_mismatch_mode = + local_context->getSettingsRef()[Setting::export_merge_tree_part_schema_mismatch_mode].value; const bool ignore_extra_source_columns_by_position = - local_context->getSettingsRef()[Setting::export_merge_tree_part_schema_mismatch_mode] - == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position; + schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position; auto source_columns = plan_for_part.getCurrentHeader()->getColumnsWithTypeAndName(); @@ -169,7 +161,9 @@ namespace auto dag = ActionsDAG::makeConvertingActions( source_columns, destination_columns, - ActionsDAG::MatchColumnsMode::Position, + schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name + ? ActionsDAG::MatchColumnsMode::Name + : ActionsDAG::MatchColumnsMode::Position, local_context); auto expression_step = std::make_unique( @@ -353,8 +347,6 @@ bool ExportPartTask::executeStep() /// This is a hack that materializes the columns before the export so they can be exported to tables that have matching columns materializeSpecialColumns(plan_for_part.getCurrentHeader(), metadata_snapshot, local_context, plan_for_part); - /// Align the pipeline header with the destination's non-materialized sample block, - /// using the same `makeConvertingActions(Position)` call INSERT SELECT performs. addExportConvertingActions(plan_for_part, *destination_storage, local_context); QueryPlanOptimizationSettings optimization_settings(local_context); diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 0c69c24511d3..ef0aa3d6fc4f 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -58,6 +58,7 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int ILLEGAL_COLUMN; extern const int NUMBER_OF_COLUMNS_DOESNT_MATCH; + extern const int THERE_IS_NO_COLUMN; extern const int INCOMPATIBLE_COLUMNS; extern const int NO_SUCH_COLUMN_IN_TABLE; extern const int FILE_ALREADY_EXISTS; @@ -112,6 +113,7 @@ namespace ExportPartitionUtils ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, ErrorCodes::ILLEGAL_COLUMN, ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, + ErrorCodes::THERE_IS_NO_COLUMN, ErrorCodes::INCOMPATIBLE_COLUMNS, ErrorCodes::NO_SUCH_COLUMN_IN_TABLE, ErrorCodes::NOT_IMPLEMENTED, @@ -733,6 +735,24 @@ namespace ExportPartitionUtils source_column.type->getName(), destination_column.type->getName()); } + + void verifyExportColumnCastIsSafe( + const ColumnWithTypeAndName & source_column, + const ColumnWithTypeAndName & destination_column, + const StorageID & destination_storage_id) + { + if (canBeSafelyCast(source_column.type, destination_column.type)) + return; + + throw Exception(ErrorCodes::INCOMPATIBLE_COLUMNS, + "Cannot export to {}: column '{}' requires a lossy cast from {} to {}, " + "which may change values. Set `export_merge_tree_part_allow_lossy_cast = 1` " + "to allow lossy casts during export.", + destination_storage_id.getFullTableName(), + destination_column.name, + source_column.type->getName(), + destination_column.type->getName()); + } } void assertPartitionKeyASTAreEqual( @@ -748,6 +768,46 @@ namespace ExportPartitionUtils throw Exception(ErrorCodes::BAD_ARGUMENTS, "Tables have different partition key"); } + void verifyExportColumnCastsAreSafe( + const ColumnsWithTypeAndName & source_columns, + const ColumnsWithTypeAndName & destination_columns, + MergeTreePartExportSchemaMismatchMode schema_mismatch_mode, + const StorageID & destination_storage_id) + { + if (schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name) + { + std::unordered_map source_columns_by_name; + source_columns_by_name.reserve(source_columns.size()); + for (const auto & source_column : source_columns) + source_columns_by_name.emplace(source_column.name, &source_column); + + for (const auto & destination_column : destination_columns) + { + const auto source_it = source_columns_by_name.find(destination_column.name); + if (source_it == source_columns_by_name.end()) + throw Exception( + ErrorCodes::THERE_IS_NO_COLUMN, + "Cannot find column `{}` in source stream", + destination_column.name); + + verifyExportColumnCastIsSafe(*source_it->second, destination_column, destination_storage_id); + } + return; + } + + if (source_columns.size() < destination_columns.size() + || (schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::strict + && source_columns.size() != destination_columns.size())) + throw Exception( + ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, + "Number of columns doesn't match (source: {} and result: {})", + source_columns.size(), + destination_columns.size()); + + for (size_t i = 0; i < destination_columns.size(); ++i) + verifyExportColumnCastIsSafe(source_columns[i], destination_columns[i], destination_storage_id); + } + void verifyExportSchemaCastable( const StorageMetadataPtr & source_metadata, const StorageMetadataPtr & destination_metadata, @@ -770,9 +830,10 @@ namespace ExportPartitionUtils /// the trimming `ExportPartTask::addExportConvertingActions` applies to the real data. /// The reverse (destination has more columns than source) is always rejected below by /// `makeConvertingActions`, in both modes. + const auto schema_mismatch_mode = + context->getSettingsRef()[Setting::export_merge_tree_part_schema_mismatch_mode].value; const bool ignore_extra_source_columns_by_position = - context->getSettingsRef()[Setting::export_merge_tree_part_schema_mismatch_mode] - == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position; + schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position; if (ignore_extra_source_columns_by_position && source_columns.size() > destination_columns.size()) { @@ -788,7 +849,9 @@ namespace ExportPartitionUtils (void) ActionsDAG::makeConvertingActions( source_columns, destination_columns, - ActionsDAG::MatchColumnsMode::Position, + schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name + ? ActionsDAG::MatchColumnsMode::Name + : ActionsDAG::MatchColumnsMode::Position, context); const auto & source_columns_description = source_metadata->getColumns(); @@ -805,29 +868,47 @@ namespace ExportPartitionUtils const bool allow_lossy_cast = context->getSettingsRef()[Setting::export_merge_tree_part_allow_lossy_cast]; - const size_t num_columns = std::min(source_columns.size(), destination_columns.size()); - for (size_t i = 0; i < num_columns; ++i) + /// Partition-key columns must keep a stable identity across the conversion, regardless of + /// `allow_lossy_cast`, so that `PARTITION BY` keeps selecting the same values after export. + /// `ignore_extra_source_columns_by_name` matches columns by name (see `makeConvertingActions` + /// above), so look the source column up by name instead of assuming the same position. + if (schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name) { - const auto & source_column = source_columns[i]; - const auto & destination_column = destination_columns[i]; + std::unordered_map source_positions_by_name; + source_positions_by_name.reserve(source_columns.size()); + for (size_t i = 0; i < source_columns.size(); ++i) + source_positions_by_name.emplace(source_columns[i].name, i); - if (partition_key_owner_columns.contains(source_column.name)) - verifyPartitionKeyColumn(source_column, destination_column, i, destination_storage_id); + for (const auto & destination_column : destination_columns) + { + if (!partition_key_owner_columns.contains(destination_column.name)) + continue; - /// Lossy casts may silently change values, so reject them unless the user opts in. - if (allow_lossy_cast) - continue; + const auto source_it = source_positions_by_name.find(destination_column.name); + if (source_it == source_positions_by_name.end()) + continue; /// Reported as THERE_IS_NO_COLUMN by verifyExportColumnCastsAreSafe below. - if (!canBeSafelyCast(source_column.type, destination_column.type)) - throw Exception(ErrorCodes::INCOMPATIBLE_COLUMNS, - "Cannot export to {}: column '{}' requires a lossy cast from {} to {}, " - "which may change values. Set `export_merge_tree_part_allow_lossy_cast = 1` " - "to allow lossy casts during export.", - destination_storage_id.getFullTableName(), - destination_column.name, - source_column.type->getName(), - destination_column.type->getName()); + verifyPartitionKeyColumn( + source_columns[source_it->second], destination_column, source_it->second, destination_storage_id); + } } + else + { + const size_t num_columns = std::min(source_columns.size(), destination_columns.size()); + for (size_t i = 0; i < num_columns; ++i) + if (partition_key_owner_columns.contains(source_columns[i].name)) + verifyPartitionKeyColumn(source_columns[i], destination_columns[i], i, destination_storage_id); + } + + /// Lossy casts may silently change values, so reject them unless the user opts in. + if (allow_lossy_cast) + return; + + verifyExportColumnCastsAreSafe( + source_columns, + destination_columns, + schema_mismatch_mode, + destination_storage_id); } } diff --git a/src/Storages/MergeTree/ExportPartitionUtils.h b/src/Storages/MergeTree/ExportPartitionUtils.h index 652ec9223394..303193072284 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.h +++ b/src/Storages/MergeTree/ExportPartitionUtils.h @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include #include #include "Storages/IStorage.h" @@ -93,15 +95,23 @@ namespace ExportPartitionUtils const StorageMetadataPtr & source_metadata, const StorageMetadataPtr & destination_metadata); + void verifyExportColumnCastsAreSafe( + const ColumnsWithTypeAndName & source_columns, + const ColumnsWithTypeAndName & destination_columns, + MergeTreePartExportSchemaMismatchMode schema_mismatch_mode, + const StorageID & destination_storage_id); + /// Validates that source columns can be exported into the destination with the - /// same positional CAST matching as `INSERT INTO dest SELECT * FROM src`. Lossy - /// casts are rejected unless `export_merge_tree_part_allow_lossy_cast` is set. + /// configured positional or name-based CAST matching. Lossy casts are rejected + /// unless `export_merge_tree_part_allow_lossy_cast` is set. /// /// By default the source and destination must have the same number of columns. /// If `export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_position'`, a /// source with more columns than the destination is allowed: the extra trailing /// source columns (by position) are excluded from the comparison here, matching /// what `ExportPartTask::addExportConvertingActions` drops from the actual data. + /// If the mode is `ignore_extra_source_columns_by_name`, destination columns are + /// matched to source columns by their exact names and may appear in a different order. /// /// Throws BAD_ARGUMENTS on any violation. void verifyExportSchemaCastable( diff --git a/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp b/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp index df20755ba590..8186db87d944 100644 --- a/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp +++ b/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp @@ -3,7 +3,10 @@ #include #include #include +#include #include +#include +#include #include #include #include @@ -11,6 +14,12 @@ namespace DB { +namespace ErrorCodes +{ + extern const int INCOMPATIBLE_COLUMNS; + extern const int THERE_IS_NO_COLUMN; +} + namespace Setting { extern const SettingsMergeTreePartExportSchemaMismatchMode export_merge_tree_part_schema_mismatch_mode; @@ -18,6 +27,27 @@ namespace Setting namespace { + template + ColumnWithTypeAndName makeColumn(const String & name) + { + auto type = std::make_shared(); + return {type->createColumn(), type, name}; + } + + template + void expectExceptionCode(Function && function, int expected_code) + { + try + { + function(); + FAIL() << "Expected exception code " << expected_code; + } + catch (const Exception & exception) + { + EXPECT_EQ(exception.code(), expected_code) << exception.message(); + } + } + ExportReplicatedMergeTreePartitionManifest makeValidManifest() { ExportReplicatedMergeTreePartitionManifest manifest; @@ -171,4 +201,84 @@ TEST_F(ExportPartitionManifestBackCompatTest, SchemaMismatchModeAppliedToWorkerC } } +TEST(ExportColumnCastsTest, UsesSelectedMatchingMode) +{ + const ColumnsWithTypeAndName source_columns = { + makeColumn("id"), + makeColumn("year"), + makeColumn("payload"), + }; + const ColumnsWithTypeAndName destination_columns = { + makeColumn("payload"), + makeColumn("year"), + makeColumn("id"), + }; + const StorageID destination_storage_id{"test", "destination"}; + + expectExceptionCode( + [&] + { + ExportPartitionUtils::verifyExportColumnCastsAreSafe( + source_columns, + destination_columns, + MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position, + destination_storage_id); + }, + ErrorCodes::INCOMPATIBLE_COLUMNS); + + EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( + source_columns, + destination_columns, + MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name, + destination_storage_id)); +} + +TEST(ExportColumnCastsTest, RejectsLossyCastAfterMatchingByName) +{ + const ColumnsWithTypeAndName source_columns = { + makeColumn("id"), + makeColumn("year"), + makeColumn("extra"), + }; + const ColumnsWithTypeAndName destination_columns = { + makeColumn("id"), + makeColumn("year"), + }; + + expectExceptionCode( + [&] + { + ExportPartitionUtils::verifyExportColumnCastsAreSafe( + source_columns, + destination_columns, + MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name, + StorageID{"test", "destination"}); + }, + ErrorCodes::INCOMPATIBLE_COLUMNS); +} + +TEST(ExportColumnCastsTest, RejectsMissingDestinationColumnAfterMatchingByName) +{ + const ColumnsWithTypeAndName source_columns = { + makeColumn("id"), + makeColumn("year"), + makeColumn("extra"), + }; + const ColumnsWithTypeAndName destination_columns = { + makeColumn("renamed_id"), + makeColumn("year"), + }; + + expectExceptionCode( + [&] + { + ExportPartitionUtils::verifyExportColumnCastsAreSafe( + source_columns, + destination_columns, + MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name, + StorageID{"test", "destination"}); + }, + ErrorCodes::THERE_IS_NO_COLUMN); +} + } diff --git a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py index 34590115d7de..b711b4734d39 100644 --- a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py +++ b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py @@ -37,6 +37,12 @@ ) +EXTRA_SOURCE_COLUMN_MODES = [ + pytest.param("ignore_extra_source_columns_by_position", id="by-position"), + pytest.param("ignore_extra_source_columns_by_name", id="by-name"), +] + + # --------------------------------------------------------------------------- # Cluster fixture # --------------------------------------------------------------------------- @@ -757,13 +763,8 @@ def test_export_part_column_count_mismatch_source_fewer_is_rejected(cluster): node.query(f"DROP TABLE IF EXISTS {iceberg}") -def test_export_part_source_more_columns_allowed_with_ignore_extra_setting(cluster): - """ - Source has 3 columns (id, year, extra), destination has 2 (id, year). - With `export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_position'`, - the export must succeed: the trailing `extra` source column is dropped - (matched positionally) and only `id`/`year` land in the destination. - """ +@pytest.mark.parametrize("mismatch_mode", EXTRA_SOURCE_COLUMN_MODES) +def test_export_part_source_more_columns_allowed_with_ignore_extra_setting(cluster, mismatch_mode): node = cluster.instances["node1"] sfx = unique_suffix() mt = f"mt_ignore_extra_{sfx}" @@ -777,7 +778,7 @@ def test_export_part_source_more_columns_allowed_with_ignore_extra_setting(clust export_part( node=node, table=mt, part=part_2020, dest=iceberg, - extra_settings="export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_position'", + extra_settings=f"export_merge_tree_part_schema_mismatch_mode = '{mismatch_mode}'", ) wait_for_export_part(node=node, table=mt, part=part_2020) @@ -793,13 +794,16 @@ def test_export_part_source_more_columns_allowed_with_ignore_extra_setting(clust node.query(f"DROP TABLE IF EXISTS {iceberg}") -def test_export_part_column_count_mismatch_source_fewer_still_rejected_with_ignore_extra_setting(cluster): - """ - `ignore_extra_source_columns_by_position` only relaxes the source-has-more-columns - direction. Source has 2 columns (id, year), destination has 3 (id, year, extra): - the destination cannot be filled from the source, so this must still be - rejected synchronously even with the relaxed setting. - """ +@pytest.mark.parametrize( + "mismatch_mode,expected_error", + [ + pytest.param("ignore_extra_source_columns_by_position", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-position"), + pytest.param("ignore_extra_source_columns_by_name", "THERE_IS_NO_COLUMN", id="by-name"), + ], +) +def test_export_part_column_count_mismatch_source_fewer_still_rejected_with_ignore_extra_setting( + cluster, mismatch_mode, expected_error +): node = cluster.instances["node1"] sfx = unique_suffix() mt = f"mt_ignore_extra_fewer_{sfx}" @@ -815,12 +819,9 @@ def test_export_part_column_count_mismatch_source_fewer_still_rejected_with_igno f"ALTER TABLE {mt} EXPORT PART '{part_2020}' TO TABLE {iceberg} " f"SETTINGS allow_experimental_export_merge_tree_part = 1, " f"allow_experimental_insert_into_iceberg = 1, " - f"export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_position'" - ) - assert "NUMBER_OF_COLUMNS_DOESNT_MATCH" in error, ( - f"Expected NUMBER_OF_COLUMNS_DOESNT_MATCH for source Date: Mon, 17 Aug 2026 14:01:21 +0200 Subject: [PATCH 02/11] remove comments --- src/Storages/MergeTree/ExportPartitionUtils.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index ef0aa3d6fc4f..86dbda30abb5 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -868,10 +868,6 @@ namespace ExportPartitionUtils const bool allow_lossy_cast = context->getSettingsRef()[Setting::export_merge_tree_part_allow_lossy_cast]; - /// Partition-key columns must keep a stable identity across the conversion, regardless of - /// `allow_lossy_cast`, so that `PARTITION BY` keeps selecting the same values after export. - /// `ignore_extra_source_columns_by_name` matches columns by name (see `makeConvertingActions` - /// above), so look the source column up by name instead of assuming the same position. if (schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name) { std::unordered_map source_positions_by_name; @@ -886,7 +882,7 @@ namespace ExportPartitionUtils const auto source_it = source_positions_by_name.find(destination_column.name); if (source_it == source_positions_by_name.end()) - continue; /// Reported as THERE_IS_NO_COLUMN by verifyExportColumnCastsAreSafe below. + continue; verifyPartitionKeyColumn( source_columns[source_it->second], destination_column, source_it->second, destination_storage_id); From cbb6348bab4ecfb5806796594cef0c6fcceb626b Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Tue, 18 Aug 2026 10:59:46 +0200 Subject: [PATCH 03/11] update text Signed-off-by: Konstantin Morozov --- src/Storages/MergeTree/ExportPartitionUtils.cpp | 16 ++++++++++------ src/Storages/MergeTree/MergeTreeData.cpp | 1 - src/Storages/StorageReplicatedMergeTree.cpp | 1 - 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 86dbda30abb5..992c0c817ca2 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -710,19 +710,21 @@ namespace ExportPartitionUtils const ColumnWithTypeAndName & source_column, const ColumnWithTypeAndName & destination_column, size_t position, - const StorageID & destination_storage_id) + const StorageID & destination_storage_id, + bool match_by_name) { if (source_column.name != destination_column.name) throw Exception( ErrorCodes::BAD_ARGUMENTS, "Cannot export to {}: partition key column '{}' is at position {} in the source " "table, but the destination's column at that position is named '{}'. EXPORT " - "PART/PARTITION matches columns by position, so partition key columns must be " - "declared at the same position in both tables.", + "PART/PARTITION {} so partition key columns must be declared {} in both tables.", destination_storage_id.getFullTableName(), source_column.name, position, - destination_column.name); + destination_column.name, + match_by_name ? "matches columns by name" : "matches columns by position", + match_by_name ? "with the same name" : "at the same position"); if (!haveSameTupleElementLayout(source_column.type, destination_column.type)) throw Exception( @@ -885,7 +887,8 @@ namespace ExportPartitionUtils continue; verifyPartitionKeyColumn( - source_columns[source_it->second], destination_column, source_it->second, destination_storage_id); + source_columns[source_it->second], destination_column, source_it->second, destination_storage_id, + /*match_by_name=*/ true); } } else @@ -893,7 +896,8 @@ namespace ExportPartitionUtils const size_t num_columns = std::min(source_columns.size(), destination_columns.size()); for (size_t i = 0; i < num_columns; ++i) if (partition_key_owner_columns.contains(source_columns[i].name)) - verifyPartitionKeyColumn(source_columns[i], destination_columns[i], i, destination_storage_id); + verifyPartitionKeyColumn(source_columns[i], destination_columns[i], i, destination_storage_id, + /*match_by_name=*/ false); } /// Lossy casts may silently change values, so reject them unless the user opts in. diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 0519730a5ee8..dfc49f1bf7bb 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -6782,7 +6782,6 @@ void MergeTreeData::exportPartToTable( #endif } - /// Positional CAST matching, like `INSERT INTO dest SELECT * FROM src`. ExportPartitionUtils::verifyExportSchemaCastable( source_metadata_ptr, destination_metadata_ptr, dest_storage->getStorageID(), query_context); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 9d40c5b35df1..ed4a82c69d7f 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -8412,7 +8412,6 @@ void StorageReplicatedMergeTree::exportPartitionToTable(const PartitionCommand & auto src_snapshot = getInMemoryMetadataPtr(); auto destination_snapshot = dest_storage->getInMemoryMetadataPtr(); - /// Positional CAST matching, like `INSERT INTO dest SELECT * FROM src`. ExportPartitionUtils::verifyExportSchemaCastable( src_snapshot, destination_snapshot, dest_storage->getStorageID(), query_context); From a38c62c1a37600e31205904e63120bc3fdf9696a Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Tue, 18 Aug 2026 13:15:12 +0200 Subject: [PATCH 04/11] by_name only for extra columns Signed-off-by: Konstantin Morozov --- docs/en/antalya/part_export.md | 6 +-- docs/en/antalya/partition_export.md | 6 +-- src/Core/Settings.cpp | 2 +- src/Storages/MergeTree/ExportPartTask.cpp | 9 +++- .../MergeTree/ExportPartitionUtils.cpp | 16 +++++-- src/Storages/MergeTree/ExportPartitionUtils.h | 6 ++- .../tests/gtest_export_partition_ordering.cpp | 45 +++++++++++++++++++ .../test.py | 2 +- .../test.py | 2 +- .../test.py | 2 +- 10 files changed, 78 insertions(+), 18 deletions(-) diff --git a/docs/en/antalya/part_export.md b/docs/en/antalya/part_export.md index fc4cefdafb51..10b15b79b9c8 100644 --- a/docs/en/antalya/part_export.md +++ b/docs/en/antalya/part_export.md @@ -143,7 +143,7 @@ In case a table function is used as the destination, the schema can be omitted a - **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` allows the source `MergeTree` table to have more columns than the destination table and how destination columns are matched in that case. Possible values: - `strict` - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`, and the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - `ignore_extra_source_columns_by_position` - the source may have more columns than the destination. The extra trailing source columns (by position) are dropped and not exported. The destination having more columns than the source is still rejected in this mode. - - `ignore_extra_source_columns_by_name` - the source may have more columns than the destination. Every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be reordered and additional source columns may occur in any position. Unmatched source columns are not exported. If a destination column is absent from the source, including when it was renamed, the export throws `THERE_IS_NO_COLUMN`; there is no positional fallback. Use `ignore_extra_source_columns_by_position` when renamed columns should be matched by position. + - `ignore_extra_source_columns_by_name` - only takes effect when the source has more columns than the destination. In that case, every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be reordered and the extra source columns may occur in any position. Unmatched source columns are not exported. If a destination column is absent from the source, including when it was renamed, the export throws `THERE_IS_NO_COLUMN`; there is no positional fallback. When the source does **not** have more columns than the destination, this mode behaves exactly like `strict` - columns are matched positionally, and a column-count mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. Use `ignore_extra_source_columns_by_position` when renamed columns should be matched by position. Extra source columns are still read and evaluated (including `MATERIALIZED`/`ALIAS` columns, and any column another kept column's `ALIAS`/`MATERIALIZED` expression depends on) before being dropped, so this setting only changes which columns end up in the destination, not what is computed while reading the part. Type conversion and `export_merge_tree_part_allow_lossy_cast` are applied after columns are matched. @@ -151,8 +151,8 @@ In case a table function is used as the destination, the schema can be omitted a - In `strict` mode, a different number of source and destination columns throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - In `ignore_extra_source_columns_by_position` mode, the destination having more columns than the source throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - - In `ignore_extra_source_columns_by_name` mode, a destination column which is absent from the source throws `THERE_IS_NO_COLUMN`. - - Renaming a destination column in `ignore_extra_source_columns_by_name` mode throws `THERE_IS_NO_COLUMN`; the mode does not fall back to positional matching. + - In `ignore_extra_source_columns_by_name` mode, when the source has more columns than the destination, a destination column absent from the source (including a renamed one) throws `THERE_IS_NO_COLUMN`; the mode does not fall back to positional matching in that case. + - In `ignore_extra_source_columns_by_name` mode, when the source does not have more columns than the destination, a column-count mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`, same as `strict`. - After columns have been matched successfully, a cast rejected by the export type-safety check throws `INCOMPATIBLE_COLUMNS`. diff --git a/docs/en/antalya/partition_export.md b/docs/en/antalya/partition_export.md index a57f38ba6ac9..934e61418c36 100644 --- a/docs/en/antalya/partition_export.md +++ b/docs/en/antalya/partition_export.md @@ -135,7 +135,7 @@ Notes: - **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` allows the source `MergeTree` table to have more columns than the destination table and how destination columns are matched in that case. Possible values: - `strict` - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`, and the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - `ignore_extra_source_columns_by_position` - the source may have more columns than the destination. The extra trailing source columns (by position) are dropped and not exported. The destination having more columns than the source is still rejected in this mode. - - `ignore_extra_source_columns_by_name` - the source may have more columns than the destination. Every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be reordered and additional source columns may occur in any position. Unmatched source columns are not exported. If a destination column is absent from the source, including when it was renamed, the export throws `THERE_IS_NO_COLUMN`; there is no positional fallback. Use `ignore_extra_source_columns_by_position` when renamed columns should be matched by position. + - `ignore_extra_source_columns_by_name` - only takes effect when the source has more columns than the destination. In that case, every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be reordered and the extra source columns may occur in any position. Unmatched source columns are not exported. If a destination column is absent from the source, including when it was renamed, the export throws `THERE_IS_NO_COLUMN`; there is no positional fallback. When the source does **not** have more columns than the destination, this mode behaves exactly like `strict` - columns are matched positionally, and a column-count mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. Use `ignore_extra_source_columns_by_position` when renamed columns should be matched by position. Extra source columns are still read and evaluated (including `MATERIALIZED`/`ALIAS` columns, and any column another kept column's `ALIAS`/`MATERIALIZED` expression depends on) before being dropped, so this setting only changes which columns end up in the destination, not what is computed while reading the part. Type conversion and `export_merge_tree_part_allow_lossy_cast` are applied after columns are matched. @@ -143,8 +143,8 @@ Notes: - In `strict` mode, a different number of source and destination columns throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - In `ignore_extra_source_columns_by_position` mode, the destination having more columns than the source throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - - In `ignore_extra_source_columns_by_name` mode, a destination column which is absent from the source throws `THERE_IS_NO_COLUMN`. - - Renaming a destination column in `ignore_extra_source_columns_by_name` mode throws `THERE_IS_NO_COLUMN`; the mode does not fall back to positional matching. + - In `ignore_extra_source_columns_by_name` mode, when the source has more columns than the destination, a destination column absent from the source (including a renamed one) throws `THERE_IS_NO_COLUMN`; the mode does not fall back to positional matching in that case. + - In `ignore_extra_source_columns_by_name` mode, when the source does not have more columns than the destination, a column-count mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`, same as `strict`. - After columns have been matched successfully, a cast rejected by the export type-safety check throws `INCOMPATIBLE_COLUMNS`. ## Examples diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 52ce73c6bf24..c9f364eeaf51 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -7632,7 +7632,7 @@ Controls whether `EXPORT PART`/`EXPORT PARTITION` allows the source `MergeTree` Possible values: - `strict` (default) - columns are matched positionally and the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - `ignore_extra_source_columns_by_position` - the source may have more columns than the destination. The extra trailing source columns (by position) are dropped and not exported. The destination having more columns than the source is still rejected in this mode. -- `ignore_extra_source_columns_by_name` - every destination column is matched to a source column with the same exact, case-sensitive name. Destination columns may be reordered and additional source columns may occur in any position. A destination column absent from the source throws `THERE_IS_NO_COLUMN`; there is no positional fallback. +- `ignore_extra_source_columns_by_name` - the source may have more columns than the destination. If it does, every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be reordered and the extra source columns may occur in any position; a destination column absent from the source throws `THERE_IS_NO_COLUMN`, with no positional fallback. When the source does not have more columns than the destination, this mode behaves exactly like `strict`: columns are matched positionally, and a mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. )", 0) \ \ /* ####################################################### */ \ diff --git a/src/Storages/MergeTree/ExportPartTask.cpp b/src/Storages/MergeTree/ExportPartTask.cpp index 8aa3511afad4..8f33e48646cb 100644 --- a/src/Storages/MergeTree/ExportPartTask.cpp +++ b/src/Storages/MergeTree/ExportPartTask.cpp @@ -128,8 +128,9 @@ namespace schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position; auto source_columns = plan_for_part.getCurrentHeader()->getColumnsWithTypeAndName(); + const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); - if (ignore_extra_source_columns_by_position && source_columns.size() > destination_columns.size()) + if (ignore_extra_source_columns_by_position && src_has_extra_columns) { LOG_DEBUG(getLogger("ExportPartTask"), "Source has {} columns while destination has {} columns, " @@ -158,10 +159,14 @@ namespace source_columns = plan_for_part.getCurrentHeader()->getColumnsWithTypeAndName(); } + const bool ignore_extra_source_columns_by_name = + schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name + && src_has_extra_columns; + auto dag = ActionsDAG::makeConvertingActions( source_columns, destination_columns, - schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name + ignore_extra_source_columns_by_name ? ActionsDAG::MatchColumnsMode::Name : ActionsDAG::MatchColumnsMode::Position, local_context); diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 992c0c817ca2..1f817afc7d1f 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -776,7 +776,10 @@ namespace ExportPartitionUtils MergeTreePartExportSchemaMismatchMode schema_mismatch_mode, const StorageID & destination_storage_id) { - if (schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name) + const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); + + if (schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name + && src_has_extra_columns) { std::unordered_map source_columns_by_name; source_columns_by_name.reserve(source_columns.size()); @@ -834,10 +837,11 @@ namespace ExportPartitionUtils /// `makeConvertingActions`, in both modes. const auto schema_mismatch_mode = context->getSettingsRef()[Setting::export_merge_tree_part_schema_mismatch_mode].value; + const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); const bool ignore_extra_source_columns_by_position = schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position; - if (ignore_extra_source_columns_by_position && source_columns.size() > destination_columns.size()) + if (ignore_extra_source_columns_by_position && src_has_extra_columns) { LOG_DEBUG(getLogger("ExportPartitionUtils"), "Source has {} columns while destination has {} columns, " @@ -848,10 +852,14 @@ namespace ExportPartitionUtils source_columns.resize(destination_columns.size()); } + const bool ignore_extra_source_columns_by_name = + schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name + && src_has_extra_columns; + (void) ActionsDAG::makeConvertingActions( source_columns, destination_columns, - schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name + ignore_extra_source_columns_by_name ? ActionsDAG::MatchColumnsMode::Name : ActionsDAG::MatchColumnsMode::Position, context); @@ -870,7 +878,7 @@ namespace ExportPartitionUtils const bool allow_lossy_cast = context->getSettingsRef()[Setting::export_merge_tree_part_allow_lossy_cast]; - if (schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name) + if (ignore_extra_source_columns_by_name) { std::unordered_map source_positions_by_name; source_positions_by_name.reserve(source_columns.size()); diff --git a/src/Storages/MergeTree/ExportPartitionUtils.h b/src/Storages/MergeTree/ExportPartitionUtils.h index 303193072284..4b8a484f7dc9 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.h +++ b/src/Storages/MergeTree/ExportPartitionUtils.h @@ -110,8 +110,10 @@ namespace ExportPartitionUtils /// source with more columns than the destination is allowed: the extra trailing /// source columns (by position) are excluded from the comparison here, matching /// what `ExportPartTask::addExportConvertingActions` drops from the actual data. - /// If the mode is `ignore_extra_source_columns_by_name`, destination columns are - /// matched to source columns by their exact names and may appear in a different order. + /// If the mode is `ignore_extra_source_columns_by_name` and the source has more columns + /// than the destination, destination columns are matched to source columns by their exact + /// names and may appear in a different order. With an equal (or smaller) column count this + /// mode has no extra source columns to ignore and behaves exactly like `strict`. /// /// Throws BAD_ARGUMENTS on any violation. void verifyExportSchemaCastable( diff --git a/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp b/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp index 8186db87d944..0dde781e4871 100644 --- a/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp +++ b/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp @@ -207,6 +207,7 @@ TEST(ExportColumnCastsTest, UsesSelectedMatchingMode) makeColumn("id"), makeColumn("year"), makeColumn("payload"), + makeColumn("extra"), }; const ColumnsWithTypeAndName destination_columns = { makeColumn("payload"), @@ -233,6 +234,50 @@ TEST(ExportColumnCastsTest, UsesSelectedMatchingMode) destination_storage_id)); } +TEST(ExportColumnCastsTest, MatchingByNameNoOpWithoutExtraSourceColumns) +{ + const ColumnsWithTypeAndName source_columns = { + makeColumn("id"), + makeColumn("year"), + makeColumn("payload"), + }; + const ColumnsWithTypeAndName reordered_destination_columns = { + makeColumn("payload"), + makeColumn("year"), + makeColumn("id"), + }; + const StorageID destination_storage_id{"test", "destination"}; + + for (const auto mode : + {MergeTreePartExportSchemaMismatchMode::strict, + MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position, + MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name}) + { + expectExceptionCode( + [&] + { + ExportPartitionUtils::verifyExportColumnCastsAreSafe( + source_columns, reordered_destination_columns, mode, destination_storage_id); + }, + ErrorCodes::INCOMPATIBLE_COLUMNS); + } + + const ColumnsWithTypeAndName same_order_destination_columns = { + makeColumn("id"), + makeColumn("year"), + makeColumn("payload"), + }; + + for (const auto mode : + {MergeTreePartExportSchemaMismatchMode::strict, + MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position, + MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name}) + { + EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( + source_columns, same_order_destination_columns, mode, destination_storage_id)); + } +} + TEST(ExportColumnCastsTest, RejectsLossyCastAfterMatchingByName) { const ColumnsWithTypeAndName source_columns = { diff --git a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py index b711b4734d39..c3a193edbd80 100644 --- a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py +++ b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py @@ -798,7 +798,7 @@ def test_export_part_source_more_columns_allowed_with_ignore_extra_setting(clust "mismatch_mode,expected_error", [ pytest.param("ignore_extra_source_columns_by_position", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-position"), - pytest.param("ignore_extra_source_columns_by_name", "THERE_IS_NO_COLUMN", id="by-name"), + pytest.param("ignore_extra_source_columns_by_name", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-name"), ], ) def test_export_part_column_count_mismatch_source_fewer_still_rejected_with_ignore_extra_setting( diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index 324521561616..24fe43c2806f 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -1040,7 +1040,7 @@ def test_export_part_column_count_mismatch_source_fewer_is_rejected(cluster): "mismatch_mode,expected_error", [ pytest.param("ignore_extra_source_columns_by_position", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-position"), - pytest.param("ignore_extra_source_columns_by_name", "THERE_IS_NO_COLUMN", id="by-name"), + pytest.param("ignore_extra_source_columns_by_name", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-name"), ], ) def test_export_part_column_count_mismatch_source_fewer_still_rejected_with_ignore_extra_setting( diff --git a/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py b/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py index 6f40f2e61373..dcccaef66009 100644 --- a/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py +++ b/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py @@ -1371,7 +1371,7 @@ def test_export_partition_source_more_columns_allowed_with_ignore_extra_setting( "mismatch_mode,expected_error", [ pytest.param("ignore_extra_source_columns_by_position", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-position"), - pytest.param("ignore_extra_source_columns_by_name", "THERE_IS_NO_COLUMN", id="by-name"), + pytest.param("ignore_extra_source_columns_by_name", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-name"), ], ) def test_export_partition_column_count_mismatch_source_fewer_still_rejected_with_ignore_extra_setting( From bf6a9f6a140fb6d183d94e9cce46309da9946bfc Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Tue, 18 Aug 2026 13:19:43 +0200 Subject: [PATCH 05/11] update tests Signed-off-by: Konstantin Morozov --- .../test.py | 43 +++++++++++++------ 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py index c3a193edbd80..157832f56547 100644 --- a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py +++ b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py @@ -1101,27 +1101,16 @@ def test_export_part_reordered_subset_requires_matching_by_name(cluster): node.query(f"DROP TABLE IF EXISTS {iceberg}") -@pytest.mark.parametrize( - "source_columns,values", - [ - pytest.param("id Int32, year Int32", "(1, 2020), (2, 2020)", id="same-column-count"), - pytest.param( - "id Int32, year Int32, extra String", - "(1, 2020, 'first'), (2, 2020, 'second')", - id="extra-source-column", - ), - ], -) -def test_export_part_match_by_name_requires_every_destination_column(cluster, source_columns, values): +def test_export_part_match_by_name_requires_every_destination_column(cluster): node = cluster.instances["node1"] sfx = unique_suffix() mt = f"mt_match_by_name_missing_{sfx}" iceberg = f"iceberg_match_by_name_missing_{sfx}" - make_mt(node, mt, source_columns, "year") + make_mt(node, mt, "id Int32, year Int32, extra String", "year") make_iceberg_s3(node, iceberg, "renamed_id Int32, year Int32", "year") - node.query(f"INSERT INTO {mt} VALUES {values}") + node.query(f"INSERT INTO {mt} VALUES (1, 2020, 'first'), (2, 2020, 'second')") part_2020 = get_part(node, mt, "2020") error = node.query_and_get_error( @@ -1140,6 +1129,32 @@ def test_export_part_match_by_name_requires_every_destination_column(cluster, so node.query(f"DROP TABLE IF EXISTS {iceberg}") +def test_export_part_match_by_name_falls_back_to_positional_without_extra_source_columns(cluster): + # No extra source column here, so `ignore_extra_source_columns_by_name` behaves like `strict`. + node = cluster.instances["node1"] + sfx = unique_suffix() + mt = f"mt_match_by_name_fallback_{sfx}" + iceberg = f"iceberg_match_by_name_fallback_{sfx}" + + make_mt(node, mt, "id Int32, year Int32", "year") + make_iceberg_s3(node, iceberg, "renamed_id Int32, year Int32", "year") + + node.query(f"INSERT INTO {mt} VALUES (1, 2020), (2, 2020)") + part_2020 = get_part(node, mt, "2020") + + export_part( + node, mt, part_2020, iceberg, + extra_settings="export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_name'", + ) + wait_for_export_part(node, mt, part_2020) + + result = node.query(f"SELECT renamed_id, year FROM {iceberg} ORDER BY renamed_id").strip() + assert result == "1\t2020\n2\t2020", f"Unexpected data:\n{result}" + + node.query(f"DROP TABLE IF EXISTS {mt} SYNC") + node.query(f"DROP TABLE IF EXISTS {iceberg}") + + def test_export_part_with_castable_widening(cluster): """ Source column is Int32, destination expects Int64. makeConvertingActions From 41b9a94ff46e7359ee616be12f2390b7a663141a Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Fri, 21 Aug 2026 18:47:42 +0200 Subject: [PATCH 06/11] change settings Signed-off-by: Konstantin Morozov --- docs/en/antalya/part_export.md | 34 ++-- docs/en/antalya/partition_export.md | 34 ++-- src/Core/Settings.cpp | 18 +- src/Core/Settings.h | 2 +- src/Core/SettingsChangesHistory.cpp | 3 +- src/Core/SettingsEnums.cpp | 2 +- src/Core/SettingsEnums.h | 9 +- ...portReplicatedMergeTreePartitionManifest.h | 28 +-- src/Storages/MergeTree/ExportPartTask.cpp | 20 +-- .../MergeTree/ExportPartitionUtils.cpp | 75 ++++---- src/Storages/MergeTree/ExportPartitionUtils.h | 22 +-- .../tests/gtest_export_partition_ordering.cpp | 164 ++++++++++++++---- src/Storages/StorageReplicatedMergeTree.cpp | 6 +- .../test.py | 100 ++++++++--- .../test.py | 95 ++++++++-- .../test.py | 73 ++++---- .../test.py | 54 +++++- 17 files changed, 521 insertions(+), 218 deletions(-) diff --git a/docs/en/antalya/part_export.md b/docs/en/antalya/part_export.md index 10b15b79b9c8..1637bf7d6a8d 100644 --- a/docs/en/antalya/part_export.md +++ b/docs/en/antalya/part_export.md @@ -49,13 +49,13 @@ SETTINGS allow_experimental_export_merge_tree_part = 1 Source and destination tables must support positional schema conversion. The following differences between the two schemas are allowed: -- **Column names** may differ between source and destination for non-partition-key columns - columns are matched by position, similar to `INSERT INTO dest SELECT * FROM src`, not by name. +- **Column names** may differ between source and destination for non-partition-key columns when `export_merge_tree_part_schema_match_mode = 'match_by_position'` (the default) - columns are matched by position, similar to `INSERT INTO dest SELECT * FROM src`, not by name. Set `export_merge_tree_part_schema_match_mode = 'match_by_name'` to match columns by their exact, case-sensitive name instead, allowing destination columns to be declared in a different order than the source. - **Column types** may differ, as long as the source type is safely castable to the destination type. Set `export_merge_tree_part_allow_lossy_cast = 1` to also permit lossy casts. - **`Tuple` element names** may differ if either the source or destination declares the tuple without named elements: an unnamed `Tuple` (e.g. `Tuple(Int32, Int32)`) is matched against the destination by element position and type only, not by name. For example, exporting from `t Tuple(Int32, Int32)` to `t Tuple(x Int32, y Int32)` is allowed as long as element types match positionally. The following must match between source and destination: -1. **Column count** - source and destination must have the same number of columns by default. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. Set `export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_position'` to allow a source table with extra trailing columns; the destination having more columns than the source is still rejected in this mode. +1. **Column count** - by default (`ignore_extra_source_columns = false`) every source column must have a corresponding destination column: with `export_merge_tree_part_schema_match_mode = 'match_by_position'` (the default) the source and destination must have the same number of columns; with `'match_by_name'` they must have the same set of column names. A mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. Set `ignore_extra_source_columns = 1` to allow a source table with columns that have no corresponding destination column; such columns are dropped and not exported. The destination having a column absent from the source is always rejected, regardless of this setting. 2. **`PARTITION BY` expressions** - for destinations other than data lakes, the source and destination `PARTITION BY` expressions must be identical. For Apache Iceberg destinations, the source partition key must be representable as an Iceberg partition spec and must match the destination partition fields and transforms. 3. **The position of every column backing the partition key** - it is not enough for the `PARTITION BY` expressions to be textually identical: every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. If such a column contains a named `Tuple`, its element names must also be declared in the same order (an unnamed `Tuple` on either side is exempt from this, per the allowance above). This comparison is recursive through nested tuples and through container types such as `Array` and `Map`. @@ -136,23 +136,31 @@ In case a table function is used as the destination, the schema can be omitted a **Warning:** A lossy cast on a partition column remains semantically truncating. For example, if a table is partitioned by an `Int64` column and some partition values do not fit into a destination `Int32` partition column, both the data files and the Iceberg metadata will contain the truncated `Int32` value (they agree with each other, but the original `Int64` value is lost). Such casts require `export_merge_tree_part_allow_lossy_cast = 1`. -### `export_merge_tree_part_schema_mismatch_mode` (Optional) +### `export_merge_tree_part_schema_match_mode` (Optional) -- **Type**: `MergeTreePartExportSchemaMismatchMode` -- **Default**: `strict` -- **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` allows the source `MergeTree` table to have more columns than the destination table and how destination columns are matched in that case. Possible values: - - `strict` - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`, and the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - - `ignore_extra_source_columns_by_position` - the source may have more columns than the destination. The extra trailing source columns (by position) are dropped and not exported. The destination having more columns than the source is still rejected in this mode. - - `ignore_extra_source_columns_by_name` - only takes effect when the source has more columns than the destination. In that case, every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be reordered and the extra source columns may occur in any position. Unmatched source columns are not exported. If a destination column is absent from the source, including when it was renamed, the export throws `THERE_IS_NO_COLUMN`; there is no positional fallback. When the source does **not** have more columns than the destination, this mode behaves exactly like `strict` - columns are matched positionally, and a column-count mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. Use `ignore_extra_source_columns_by_position` when renamed columns should be matched by position. +- **Type**: `MergeTreePartExportSchemaMatchMode` +- **Default**: `match_by_position` +- **Description**: Controls how `EXPORT PART`/`EXPORT PARTITION` matches source `MergeTree` columns to destination columns. Possible values: + - `match_by_position` (default) - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. Column names are not otherwise considered. + - `match_by_name` - every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be declared in a different order than the source. A destination column absent from the source, including when it was renamed, throws `THERE_IS_NO_COLUMN`; there is no positional fallback. + + See `ignore_extra_source_columns` below for how a source column without a corresponding destination column is handled in each mode. + +### `ignore_extra_source_columns` (Optional) + +- **Type**: `Bool` +- **Default**: `false` +- **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` tolerates source `MergeTree` columns that have no corresponding destination column. + - `false` (default) - such a source column is rejected: the source and destination must match exactly. With `export_merge_tree_part_schema_match_mode = 'match_by_position'` this means the same number of columns; with `'match_by_name'` this means the same set of column names, so a source table with columns absent from the destination is rejected even if the matched columns would otherwise be compatible. A mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. + - `true` - a source column without a corresponding destination column is dropped and not exported, instead of throwing. In `match_by_position` mode, this allows a source with extra trailing columns (the destination having more columns than the source is still always rejected). In `match_by_name` mode, this allows source columns whose name has no destination counterpart, which may occur in any position. Extra source columns are still read and evaluated (including `MATERIALIZED`/`ALIAS` columns, and any column another kept column's `ALIAS`/`MATERIALIZED` expression depends on) before being dropped, so this setting only changes which columns end up in the destination, not what is computed while reading the part. Type conversion and `export_merge_tree_part_allow_lossy_cast` are applied after columns are matched. Error behavior: - - In `strict` mode, a different number of source and destination columns throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - - In `ignore_extra_source_columns_by_position` mode, the destination having more columns than the source throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - - In `ignore_extra_source_columns_by_name` mode, when the source has more columns than the destination, a destination column absent from the source (including a renamed one) throws `THERE_IS_NO_COLUMN`; the mode does not fall back to positional matching in that case. - - In `ignore_extra_source_columns_by_name` mode, when the source does not have more columns than the destination, a column-count mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`, same as `strict`. + - With `ignore_extra_source_columns = false` (default), a source column without a corresponding destination column throws `NUMBER_OF_COLUMNS_DOESNT_MATCH` - in `match_by_position` mode this means any column-count mismatch, in `match_by_name` mode this means the source and destination column-name sets differ. + - In `match_by_position` mode, the destination having more columns than the source always throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`, regardless of `ignore_extra_source_columns`. + - In `match_by_name` mode, a destination column absent from the source (including a renamed one) always throws `THERE_IS_NO_COLUMN`, regardless of `ignore_extra_source_columns`; there is no positional fallback. - After columns have been matched successfully, a cast rejected by the export type-safety check throws `INCOMPATIBLE_COLUMNS`. diff --git a/docs/en/antalya/partition_export.md b/docs/en/antalya/partition_export.md index 934e61418c36..17b42b91f66c 100644 --- a/docs/en/antalya/partition_export.md +++ b/docs/en/antalya/partition_export.md @@ -45,9 +45,9 @@ TO TABLE [destination_database.]destination_table ## Requirements -`EXPORT PARTITION` exports each part via the same mechanism as [`EXPORT PART`](/docs/en/antalya/part_export.md#requirements), so the source and destination tables must satisfy the same compatibility requirements. Column names may differ (columns are matched by position, not by name), and column types may differ as long as they are safely castable (or `export_merge_tree_part_allow_lossy_cast = 1` is set). Beyond that, the following must match: +`EXPORT PARTITION` exports each part via the same mechanism as [`EXPORT PART`](/docs/en/antalya/part_export.md#requirements), so the source and destination tables must satisfy the same compatibility requirements. Columns are matched by position by default, or by their exact, case-sensitive name if `export_merge_tree_part_schema_match_mode = 'match_by_name'` is set, and column types may differ as long as they are safely castable (or `export_merge_tree_part_allow_lossy_cast = 1` is set). Beyond that, the following must match: -1. **Column count** - source and destination must have the same number of columns by default. Set `export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_position'` to allow a source table with extra trailing columns; the destination having more columns than the source is still rejected in this mode. +1. **Column count** - by default (`ignore_extra_source_columns = false`) every source column must have a corresponding destination column: with `export_merge_tree_part_schema_match_mode = 'match_by_position'` (the default) the source and destination must have the same number of columns; with `'match_by_name'` they must have the same set of column names. Set `ignore_extra_source_columns = 1` to allow a source table with columns that have no corresponding destination column; the destination having a column absent from the source is still always rejected. 2. **`PARTITION BY` expressions** - for destinations other than data lakes, the source and destination `PARTITION BY` expressions must be identical. For Apache Iceberg destinations, the source partition key must match the destination partition fields and transforms. 3. **Partition key column positions and layouts** - every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. Named `Tuple` elements within such a column must also be declared in the same order, including tuples nested inside `Array` or `Map`. This applies even if both tables' `PARTITION BY` expressions are textually identical. See [`EXPORT PART` requirements](/docs/en/antalya/part_export.md#requirements) for a worked example and the corresponding exception message. @@ -128,23 +128,31 @@ Notes: **Warning:** A lossy cast on a partition column remains semantically truncating. For example, if a table is partitioned by an `Int64` column and some partition values do not fit into a destination `Int32` partition column, both the data files and the Iceberg metadata will contain the truncated `Int32` value (they agree with each other, but the original `Int64` value is lost). Such casts require `export_merge_tree_part_allow_lossy_cast = 1`. -### `export_merge_tree_part_schema_mismatch_mode` (Optional) +### `export_merge_tree_part_schema_match_mode` (Optional) -- **Type**: `MergeTreePartExportSchemaMismatchMode` -- **Default**: `strict` -- **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` allows the source `MergeTree` table to have more columns than the destination table and how destination columns are matched in that case. Possible values: - - `strict` - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`, and the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - - `ignore_extra_source_columns_by_position` - the source may have more columns than the destination. The extra trailing source columns (by position) are dropped and not exported. The destination having more columns than the source is still rejected in this mode. - - `ignore_extra_source_columns_by_name` - only takes effect when the source has more columns than the destination. In that case, every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be reordered and the extra source columns may occur in any position. Unmatched source columns are not exported. If a destination column is absent from the source, including when it was renamed, the export throws `THERE_IS_NO_COLUMN`; there is no positional fallback. When the source does **not** have more columns than the destination, this mode behaves exactly like `strict` - columns are matched positionally, and a column-count mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. Use `ignore_extra_source_columns_by_position` when renamed columns should be matched by position. +- **Type**: `MergeTreePartExportSchemaMatchMode` +- **Default**: `match_by_position` +- **Description**: Controls how `EXPORT PART`/`EXPORT PARTITION` matches source `MergeTree` columns to destination columns. Possible values: + - `match_by_position` (default) - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. Column names are not otherwise considered. + - `match_by_name` - every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be declared in a different order than the source. A destination column absent from the source, including when it was renamed, throws `THERE_IS_NO_COLUMN`; there is no positional fallback. + + See `ignore_extra_source_columns` below for how a source column without a corresponding destination column is handled in each mode. + +### `ignore_extra_source_columns` (Optional) + +- **Type**: `Bool` +- **Default**: `false` +- **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` tolerates source `MergeTree` columns that have no corresponding destination column. + - `false` (default) - such a source column is rejected: the source and destination must match exactly. With `export_merge_tree_part_schema_match_mode = 'match_by_position'` this means the same number of columns; with `'match_by_name'` this means the same set of column names, so a source table with columns absent from the destination is rejected even if the matched columns would otherwise be compatible. A mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. + - `true` - a source column without a corresponding destination column is dropped and not exported, instead of throwing. In `match_by_position` mode, this allows a source with extra trailing columns (the destination having more columns than the source is still always rejected). In `match_by_name` mode, this allows source columns whose name has no destination counterpart, which may occur in any position. Extra source columns are still read and evaluated (including `MATERIALIZED`/`ALIAS` columns, and any column another kept column's `ALIAS`/`MATERIALIZED` expression depends on) before being dropped, so this setting only changes which columns end up in the destination, not what is computed while reading the part. Type conversion and `export_merge_tree_part_allow_lossy_cast` are applied after columns are matched. Error behavior: - - In `strict` mode, a different number of source and destination columns throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - - In `ignore_extra_source_columns_by_position` mode, the destination having more columns than the source throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - - In `ignore_extra_source_columns_by_name` mode, when the source has more columns than the destination, a destination column absent from the source (including a renamed one) throws `THERE_IS_NO_COLUMN`; the mode does not fall back to positional matching in that case. - - In `ignore_extra_source_columns_by_name` mode, when the source does not have more columns than the destination, a column-count mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`, same as `strict`. + - With `ignore_extra_source_columns = false` (default), a source column without a corresponding destination column throws `NUMBER_OF_COLUMNS_DOESNT_MATCH` - in `match_by_position` mode this means any column-count mismatch, in `match_by_name` mode this means the source and destination column-name sets differ. + - In `match_by_position` mode, the destination having more columns than the source always throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`, regardless of `ignore_extra_source_columns`. + - In `match_by_name` mode, a destination column absent from the source (including a renamed one) always throws `THERE_IS_NO_COLUMN`, regardless of `ignore_extra_source_columns`; there is no positional fallback. - After columns have been matched successfully, a cast rejected by the export type-safety check throws `INCOMPATIBLE_COLUMNS`. ## Examples diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index c9f364eeaf51..45eb39385754 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -7627,12 +7627,20 @@ Allow `EXPORT PART`/`EXPORT PARTITION` to apply lossy (non-value-preserving) cas When exporting to Apache Iceberg, the partition value written to the metadata is derived from the source partition columns by casting them to the destination partition-field types and applying the destination partition transform — the same computation the exported data files use, so the metadata stays consistent with the data. A lossy cast on a partition column remains semantically truncating: both the data files and the metadata contain the truncated value, and such casts require this setting to be enabled. )", 0) \ - DECLARE(MergeTreePartExportSchemaMismatchMode, export_merge_tree_part_schema_mismatch_mode, MergeTreePartExportSchemaMismatchMode::strict, R"( -Controls whether `EXPORT PART`/`EXPORT PARTITION` allows the source `MergeTree` table to have more columns than the destination table and how destination columns are matched. + DECLARE(MergeTreePartExportSchemaMatchMode, export_merge_tree_part_schema_match_mode, MergeTreePartExportSchemaMatchMode::match_by_position, R"( +Controls how `EXPORT PART`/`EXPORT PARTITION` matches source `MergeTree` columns to destination columns. Possible values: -- `strict` (default) - columns are matched positionally and the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. -- `ignore_extra_source_columns_by_position` - the source may have more columns than the destination. The extra trailing source columns (by position) are dropped and not exported. The destination having more columns than the source is still rejected in this mode. -- `ignore_extra_source_columns_by_name` - the source may have more columns than the destination. If it does, every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be reordered and the extra source columns may occur in any position; a destination column absent from the source throws `THERE_IS_NO_COLUMN`, with no positional fallback. When the source does not have more columns than the destination, this mode behaves exactly like `strict`: columns are matched positionally, and a mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. +- `match_by_position` (default) - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. Column names are not otherwise considered. +- `match_by_name` - every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be declared in a different order than the source. A destination column absent from the source throws `THERE_IS_NO_COLUMN`, with no positional fallback. + +See also `ignore_extra_source_columns`, which controls whether a source column without a corresponding destination column is dropped or rejected. +)", 0) \ + DECLARE(Bool, ignore_extra_source_columns, false, R"( +Controls whether `EXPORT PART`/`EXPORT PARTITION` tolerates source `MergeTree` columns that have no corresponding destination column. +- `false` (default) - such a source column is rejected: the source and destination must match exactly (in `export_merge_tree_part_schema_match_mode = 'match_by_position'`, this means the same number of columns; in `'match_by_name'`, the same set of column names). A mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. +- `true` - a source column without a corresponding destination column is dropped and not exported, instead of throwing. In `match_by_position` mode, this allows a source with extra trailing columns (the destination having more columns than the source is still always rejected). In `match_by_name` mode, this allows source columns whose name has no destination counterpart. + +Extra source columns are still read and evaluated (including `MATERIALIZED`/`ALIAS` columns, and any column another kept column's `ALIAS`/`MATERIALIZED` expression depends on) before being dropped, so this setting only changes which columns end up in the destination, not what is computed while reading the part. )", 0) \ \ /* ####################################################### */ \ diff --git a/src/Core/Settings.h b/src/Core/Settings.h index a5fef6160201..a166435fcd95 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -85,7 +85,7 @@ class WriteBuffer; M(CLASS_NAME, Map) \ M(CLASS_NAME, MaxThreads) \ M(CLASS_NAME, MergeTreePartExportFileAlreadyExistsPolicy) \ - M(CLASS_NAME, MergeTreePartExportSchemaMismatchMode) \ + M(CLASS_NAME, MergeTreePartExportSchemaMatchMode) \ M(CLASS_NAME, Milliseconds) \ M(CLASS_NAME, MsgPackUUIDRepresentation) \ M(CLASS_NAME, MySQLDataTypesSupport) \ diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 60b9114149a6..580d09fc8703 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -44,7 +44,8 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"object_storage_cluster_join_mode", "allow", "allow", "New setting"}, {"export_merge_tree_partition_task_timeout_seconds", "3600", "86400", "Increase default value to make it more realistic"}, {"export_merge_tree_part_allow_lossy_cast", false, false, "New setting to gate lossy casts in EXPORT PART/PARTITION behind explicit acknowledgment"}, - {"export_merge_tree_part_schema_mismatch_mode", "strict", "strict", "New setting to allow EXPORT PART/EXPORT PARTITION when the source table has more columns than the destination"}, + {"export_merge_tree_part_schema_match_mode", "match_by_position", "match_by_position", "New setting to control how EXPORT PART/EXPORT PARTITION matches source columns to destination columns"}, + {"ignore_extra_source_columns", false, false, "New setting to allow EXPORT PART/EXPORT PARTITION when the source table has columns absent from the destination"}, {"export_merge_tree_partition_retry_initial_backoff_seconds", 5, 5, "New setting for exponential back-off between failed part export retries in an export partition task"}, {"export_merge_tree_partition_retry_max_backoff_seconds", 300, 300, "New setting capping the exponential back-off between failed part export retries in an export partition task"}, {"export_merge_tree_partition_max_retries", 3, 3, "Obsolete and ignored: export partition tasks now retry retryable failures until the task timeout and fail immediately on non-retryable errors, instead of using a fixed retry budget"}, diff --git a/src/Core/SettingsEnums.cpp b/src/Core/SettingsEnums.cpp index 9aadee2db780..351f8f6be7a2 100644 --- a/src/Core/SettingsEnums.cpp +++ b/src/Core/SettingsEnums.cpp @@ -483,7 +483,7 @@ IMPLEMENT_SETTING_ENUM(JemallocProfileFormat, ErrorCodes::BAD_ARGUMENTS, IMPLEMENT_SETTING_AUTO_ENUM(MergeTreePartExportFileAlreadyExistsPolicy, ErrorCodes::BAD_ARGUMENTS); -IMPLEMENT_SETTING_AUTO_ENUM(MergeTreePartExportSchemaMismatchMode, ErrorCodes::BAD_ARGUMENTS); +IMPLEMENT_SETTING_AUTO_ENUM(MergeTreePartExportSchemaMatchMode, ErrorCodes::BAD_ARGUMENTS); IMPLEMENT_SETTING_AUTO_ENUM(ExportPartitionAllOnError, ErrorCodes::BAD_ARGUMENTS); diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index f5c48509e0b0..a325835452d5 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -574,14 +574,13 @@ enum class MergeTreePartExportFileAlreadyExistsPolicy : uint8_t DECLARE_SETTING_ENUM(MergeTreePartExportFileAlreadyExistsPolicy) -enum class MergeTreePartExportSchemaMismatchMode : uint8_t +enum class MergeTreePartExportSchemaMatchMode : uint8_t { - strict, - ignore_extra_source_columns_by_position, - ignore_extra_source_columns_by_name, + match_by_position, + match_by_name, }; -DECLARE_SETTING_ENUM(MergeTreePartExportSchemaMismatchMode) +DECLARE_SETTING_ENUM(MergeTreePartExportSchemaMatchMode) enum class ExportPartitionAllOnError : uint8_t { diff --git a/src/Storages/ExportReplicatedMergeTreePartitionManifest.h b/src/Storages/ExportReplicatedMergeTreePartitionManifest.h index 753095900fa8..791691b849da 100644 --- a/src/Storages/ExportReplicatedMergeTreePartitionManifest.h +++ b/src/Storages/ExportReplicatedMergeTreePartitionManifest.h @@ -248,7 +248,8 @@ struct ExportReplicatedMergeTreePartitionManifest std::optional output_format_compression_level; std::optional parquet_row_group_size; std::optional parquet_row_group_size_bytes; - std::optional schema_mismatch_mode; + std::optional schema_match_mode; + std::optional ignore_extra_source_columns; std::string toJsonString() const { @@ -291,8 +292,10 @@ struct ExportReplicatedMergeTreePartitionManifest json.set("parquet_row_group_size", *parquet_row_group_size); if (parquet_row_group_size_bytes) json.set("parquet_row_group_size_bytes", *parquet_row_group_size_bytes); - if (schema_mismatch_mode) - json.set("schema_mismatch_mode", String(magic_enum::enum_name(*schema_mismatch_mode))); + if (schema_match_mode) + json.set("schema_match_mode", String(magic_enum::enum_name(*schema_match_mode))); + if (ignore_extra_source_columns) + json.set("ignore_extra_source_columns", *ignore_extra_source_columns); std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM oss.exceptions(std::ios::failbit); Poco::JSON::Stringifier::stringify(json, oss); @@ -360,15 +363,20 @@ struct ExportReplicatedMergeTreePartitionManifest /// on upgrade. New tasks always persist the initiator's actual choice. manifest.allow_lossy_cast = json->has("allow_lossy_cast") ? json->getValue("allow_lossy_cast") : true; - /// Left unset (nullopt) for tasks created before this field existed - such tasks were - /// always scheduled under the old, strict column-count check (a mismatch could never + /// Left unset (nullopt) for tasks created before these fields existed - such tasks were + /// always scheduled under the old, strict column-matching check (a mismatch could never /// reach scheduling in the first place), so callers should treat an absent value as - /// `strict`. - if (json->has("schema_mismatch_mode")) + /// `match_by_position` with `ignore_extra_source_columns = false`. + if (json->has("schema_match_mode")) { - const auto schema_mismatch_mode = magic_enum::enum_cast(json->getValue("schema_mismatch_mode")); - if (schema_mismatch_mode) - manifest.schema_mismatch_mode = schema_mismatch_mode; + const auto schema_match_mode = magic_enum::enum_cast(json->getValue("schema_match_mode")); + if (schema_match_mode) + manifest.schema_match_mode = schema_match_mode; + } + + if (json->has("ignore_extra_source_columns")) + { + manifest.ignore_extra_source_columns = json->getValue("ignore_extra_source_columns"); } if (json->has("parquet_compression_method")) diff --git a/src/Storages/MergeTree/ExportPartTask.cpp b/src/Storages/MergeTree/ExportPartTask.cpp index 8f33e48646cb..2126601ce7c3 100644 --- a/src/Storages/MergeTree/ExportPartTask.cpp +++ b/src/Storages/MergeTree/ExportPartTask.cpp @@ -65,7 +65,8 @@ namespace Setting extern const SettingsUInt64 export_merge_tree_part_max_rows_per_file; extern const SettingsBool allow_experimental_analyzer; extern const SettingsString export_merge_tree_part_filename_pattern; - extern const SettingsMergeTreePartExportSchemaMismatchMode export_merge_tree_part_schema_mismatch_mode; + extern const SettingsMergeTreePartExportSchemaMatchMode export_merge_tree_part_schema_match_mode; + extern const SettingsBool ignore_extra_source_columns; } namespace @@ -122,15 +123,16 @@ namespace = destination_storage.getInMemoryMetadataPtr()->getSampleBlockNonMaterialized(); const auto & destination_columns = destination_header.getColumnsWithTypeAndName(); - const auto schema_mismatch_mode = - local_context->getSettingsRef()[Setting::export_merge_tree_part_schema_mismatch_mode].value; - const bool ignore_extra_source_columns_by_position = - schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position; + const auto schema_match_mode = + local_context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value; + const bool ignore_extra_source_columns = + local_context->getSettingsRef()[Setting::ignore_extra_source_columns]; + const bool match_by_name = schema_match_mode == MergeTreePartExportSchemaMatchMode::match_by_name; auto source_columns = plan_for_part.getCurrentHeader()->getColumnsWithTypeAndName(); const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); - if (ignore_extra_source_columns_by_position && src_has_extra_columns) + if (!match_by_name && ignore_extra_source_columns && src_has_extra_columns) { LOG_DEBUG(getLogger("ExportPartTask"), "Source has {} columns while destination has {} columns, " @@ -159,14 +161,10 @@ namespace source_columns = plan_for_part.getCurrentHeader()->getColumnsWithTypeAndName(); } - const bool ignore_extra_source_columns_by_name = - schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name - && src_has_extra_columns; - auto dag = ActionsDAG::makeConvertingActions( source_columns, destination_columns, - ignore_extra_source_columns_by_name + match_by_name ? ActionsDAG::MatchColumnsMode::Name : ActionsDAG::MatchColumnsMode::Position, local_context); diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 1f817afc7d1f..d19bb423c6c7 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -84,7 +84,8 @@ namespace ErrorCodes namespace Setting { extern const SettingsBool export_merge_tree_part_allow_lossy_cast; - extern const SettingsMergeTreePartExportSchemaMismatchMode export_merge_tree_part_schema_mismatch_mode; + extern const SettingsMergeTreePartExportSchemaMatchMode export_merge_tree_part_schema_match_mode; + extern const SettingsBool ignore_extra_source_columns; } namespace FailPoints @@ -175,12 +176,16 @@ namespace ExportPartitionUtils context_copy->setSetting("output_format_parquet_row_group_size", *manifest.parquet_row_group_size); if (manifest.parquet_row_group_size_bytes) context_copy->setSetting("output_format_parquet_row_group_size_bytes", *manifest.parquet_row_group_size_bytes); - /// Manifests written before this setting existed have no value here; such tasks were always - /// scheduled under the old, strict column-count check, so an absent value must resolve to - /// `strict` regardless of the ambient context's setting (which may have since been changed). + /// Manifests written before these settings existed have no value here; such tasks were always + /// scheduled under the old, strict column-matching check, so an absent value must resolve to + /// `match_by_position` / `false` regardless of the ambient context's settings (which may have + /// since been changed). context_copy->setSetting( - "export_merge_tree_part_schema_mismatch_mode", - String(magic_enum::enum_name(manifest.schema_mismatch_mode.value_or(MergeTreePartExportSchemaMismatchMode::strict)))); + "export_merge_tree_part_schema_match_mode", + String(magic_enum::enum_name(manifest.schema_match_mode.value_or(MergeTreePartExportSchemaMatchMode::match_by_position)))); + context_copy->setSetting( + "ignore_extra_source_columns", + manifest.ignore_extra_source_columns.value_or(false)); context_copy->setSetting("max_threads", manifest.max_threads); context_copy->setSetting("export_merge_tree_part_file_already_exists_policy", String(magic_enum::enum_name(manifest.file_already_exists_policy))); @@ -773,14 +778,23 @@ namespace ExportPartitionUtils void verifyExportColumnCastsAreSafe( const ColumnsWithTypeAndName & source_columns, const ColumnsWithTypeAndName & destination_columns, - MergeTreePartExportSchemaMismatchMode schema_mismatch_mode, + MergeTreePartExportSchemaMatchMode schema_match_mode, + bool ignore_extra_source_columns, const StorageID & destination_storage_id) { - const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); - - if (schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name - && src_has_extra_columns) + if (schema_match_mode == MergeTreePartExportSchemaMatchMode::match_by_name) { + /// `ignore_extra_source_columns` only excuses the source having MORE columns than the + /// destination; a source with fewer columns is always a mismatch. + const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); + if (source_columns.size() != destination_columns.size() + && !(ignore_extra_source_columns && src_has_extra_columns)) + throw Exception( + ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, + "Number of columns doesn't match (source: {} and result: {})", + source_columns.size(), + destination_columns.size()); + std::unordered_map source_columns_by_name; source_columns_by_name.reserve(source_columns.size()); for (const auto & source_column : source_columns) @@ -801,8 +815,7 @@ namespace ExportPartitionUtils } if (source_columns.size() < destination_columns.size() - || (schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::strict - && source_columns.size() != destination_columns.size())) + || (!ignore_extra_source_columns && source_columns.size() != destination_columns.size())) throw Exception( ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, "Number of columns doesn't match (source: {} and result: {})", @@ -830,18 +843,14 @@ namespace ExportPartitionUtils auto source_columns = source_sample_block.getColumnsWithTypeAndName(); const auto & destination_columns = destination_sample_block.getColumnsWithTypeAndName(); - /// In `ignore_extra_source_columns_by_position` mode a source with more columns than the destination - /// is allowed: the extra trailing source columns (by position) are dropped, mirroring - /// the trimming `ExportPartTask::addExportConvertingActions` applies to the real data. - /// The reverse (destination has more columns than source) is always rejected below by - /// `makeConvertingActions`, in both modes. - const auto schema_mismatch_mode = - context->getSettingsRef()[Setting::export_merge_tree_part_schema_mismatch_mode].value; + const auto schema_match_mode = + context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value; + const bool ignore_extra_source_columns = + context->getSettingsRef()[Setting::ignore_extra_source_columns]; + const bool match_by_name = schema_match_mode == MergeTreePartExportSchemaMatchMode::match_by_name; const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); - const bool ignore_extra_source_columns_by_position = - schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position; - if (ignore_extra_source_columns_by_position && src_has_extra_columns) + if (!match_by_name && ignore_extra_source_columns && src_has_extra_columns) { LOG_DEBUG(getLogger("ExportPartitionUtils"), "Source has {} columns while destination has {} columns, " @@ -852,14 +861,21 @@ namespace ExportPartitionUtils source_columns.resize(destination_columns.size()); } - const bool ignore_extra_source_columns_by_name = - schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name - && src_has_extra_columns; + /// `makeConvertingActions` in `Name` mode silently ignores an unreferenced source column, so it must be rejected here explicitly. + /// `ignore_extra_source_columns` only excuses the source having MORE columns than the destination; + /// a source with fewer columns is always a mismatch. + if (match_by_name && source_columns.size() != destination_columns.size() + && !(ignore_extra_source_columns && src_has_extra_columns)) + throw Exception( + ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, + "Number of columns doesn't match (source: {} and result: {})", + source_columns.size(), + destination_columns.size()); (void) ActionsDAG::makeConvertingActions( source_columns, destination_columns, - ignore_extra_source_columns_by_name + match_by_name ? ActionsDAG::MatchColumnsMode::Name : ActionsDAG::MatchColumnsMode::Position, context); @@ -878,7 +894,7 @@ namespace ExportPartitionUtils const bool allow_lossy_cast = context->getSettingsRef()[Setting::export_merge_tree_part_allow_lossy_cast]; - if (ignore_extra_source_columns_by_name) + if (match_by_name) { std::unordered_map source_positions_by_name; source_positions_by_name.reserve(source_columns.size()); @@ -915,7 +931,8 @@ namespace ExportPartitionUtils verifyExportColumnCastsAreSafe( source_columns, destination_columns, - schema_mismatch_mode, + schema_match_mode, + ignore_extra_source_columns, destination_storage_id); } } diff --git a/src/Storages/MergeTree/ExportPartitionUtils.h b/src/Storages/MergeTree/ExportPartitionUtils.h index 4b8a484f7dc9..a56377728b79 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.h +++ b/src/Storages/MergeTree/ExportPartitionUtils.h @@ -98,24 +98,14 @@ namespace ExportPartitionUtils void verifyExportColumnCastsAreSafe( const ColumnsWithTypeAndName & source_columns, const ColumnsWithTypeAndName & destination_columns, - MergeTreePartExportSchemaMismatchMode schema_mismatch_mode, + MergeTreePartExportSchemaMatchMode schema_match_mode, + bool ignore_extra_source_columns, const StorageID & destination_storage_id); - /// Validates that source columns can be exported into the destination with the - /// configured positional or name-based CAST matching. Lossy casts are rejected - /// unless `export_merge_tree_part_allow_lossy_cast` is set. - /// - /// By default the source and destination must have the same number of columns. - /// If `export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_position'`, a - /// source with more columns than the destination is allowed: the extra trailing - /// source columns (by position) are excluded from the comparison here, matching - /// what `ExportPartTask::addExportConvertingActions` drops from the actual data. - /// If the mode is `ignore_extra_source_columns_by_name` and the source has more columns - /// than the destination, destination columns are matched to source columns by their exact - /// names and may appear in a different order. With an equal (or smaller) column count this - /// mode has no extra source columns to ignore and behaves exactly like `strict`. - /// - /// Throws BAD_ARGUMENTS on any violation. + /// Validates that source columns can be exported into the destination with the configured + /// positional or name-based CAST matching (`export_merge_tree_part_schema_match_mode`) and + /// unmatched-column policy (`ignore_extra_source_columns`). Lossy casts are rejected unless + /// `export_merge_tree_part_allow_lossy_cast` is set. Throws BAD_ARGUMENTS on any violation. void verifyExportSchemaCastable( const StorageMetadataPtr & source_metadata, const StorageMetadataPtr & destination_metadata, diff --git a/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp b/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp index 0dde781e4871..71206642da1a 100644 --- a/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp +++ b/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp @@ -18,11 +18,13 @@ namespace ErrorCodes { extern const int INCOMPATIBLE_COLUMNS; extern const int THERE_IS_NO_COLUMN; + extern const int NUMBER_OF_COLUMNS_DOESNT_MATCH; } namespace Setting { - extern const SettingsMergeTreePartExportSchemaMismatchMode export_merge_tree_part_schema_mismatch_mode; + extern const SettingsMergeTreePartExportSchemaMatchMode export_merge_tree_part_schema_match_mode; + extern const SettingsBool ignore_extra_source_columns; } namespace @@ -144,63 +146,112 @@ TEST_F(ExportPartitionOrderingTest, IterationOrderMatchesCreateTime) } -TEST_F(ExportPartitionManifestBackCompatTest, MissingSchemaMismatchModeParsesAsNullopt) +TEST_F(ExportPartitionManifestBackCompatTest, MissingSchemaMatchModeParsesAsNullopt) { auto manifest = makeValidManifest(); - manifest.schema_mismatch_mode = MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position; + manifest.schema_match_mode = MergeTreePartExportSchemaMatchMode::match_by_name; Poco::JSON::Parser parser; auto json = parser.parse(manifest.toJsonString()).extract(); - json->remove("schema_mismatch_mode"); + json->remove("schema_match_mode"); std::ostringstream oss; oss.exceptions(std::ios::failbit); Poco::JSON::Stringifier::stringify(json, oss); auto parsed = ExportReplicatedMergeTreePartitionManifest::fromJsonString(oss.str()); - EXPECT_FALSE(parsed.schema_mismatch_mode.has_value()); + EXPECT_FALSE(parsed.schema_match_mode.has_value()); } -TEST_F(ExportPartitionManifestBackCompatTest, SchemaMismatchModeRoundTripsForEveryValue) +TEST_F(ExportPartitionManifestBackCompatTest, SchemaMatchModeRoundTripsForEveryValue) { - for (const auto value : magic_enum::enum_values()) + for (const auto value : magic_enum::enum_values()) { auto manifest = makeValidManifest(); - manifest.schema_mismatch_mode = value; + manifest.schema_match_mode = value; auto parsed = ExportReplicatedMergeTreePartitionManifest::fromJsonString(manifest.toJsonString()); - ASSERT_TRUE(parsed.schema_mismatch_mode.has_value()) << "value=" << magic_enum::enum_name(value); - EXPECT_EQ(*parsed.schema_mismatch_mode, value) << "value=" << magic_enum::enum_name(value); + ASSERT_TRUE(parsed.schema_match_mode.has_value()) << "value=" << magic_enum::enum_name(value); + EXPECT_EQ(*parsed.schema_match_mode, value) << "value=" << magic_enum::enum_name(value); } } -TEST_F(ExportPartitionManifestBackCompatTest, MissingSchemaMismatchModeFallsBackToStrictInWorkerContext) +TEST_F(ExportPartitionManifestBackCompatTest, MissingIgnoreExtraSourceColumnsParsesAsNullopt) { auto manifest = makeValidManifest(); - ASSERT_FALSE(manifest.schema_mismatch_mode.has_value()); + manifest.ignore_extra_source_columns = true; + + Poco::JSON::Parser parser; + auto json = parser.parse(manifest.toJsonString()).extract(); + json->remove("ignore_extra_source_columns"); + std::ostringstream oss; + oss.exceptions(std::ios::failbit); + Poco::JSON::Stringifier::stringify(json, oss); + + auto parsed = ExportReplicatedMergeTreePartitionManifest::fromJsonString(oss.str()); + EXPECT_FALSE(parsed.ignore_extra_source_columns.has_value()); +} + +TEST_F(ExportPartitionManifestBackCompatTest, IgnoreExtraSourceColumnsRoundTripsForEveryValue) +{ + for (const bool value : {false, true}) + { + auto manifest = makeValidManifest(); + manifest.ignore_extra_source_columns = value; + + auto parsed = ExportReplicatedMergeTreePartitionManifest::fromJsonString(manifest.toJsonString()); + + ASSERT_TRUE(parsed.ignore_extra_source_columns.has_value()) << "value=" << value; + EXPECT_EQ(*parsed.ignore_extra_source_columns, value) << "value=" << value; + } +} + +TEST_F(ExportPartitionManifestBackCompatTest, MissingSchemaMatchSettingsFallBackToDefaultsInWorkerContext) +{ + auto manifest = makeValidManifest(); + ASSERT_FALSE(manifest.schema_match_mode.has_value()); + ASSERT_FALSE(manifest.ignore_extra_source_columns.has_value()); auto worker_context = ExportPartitionUtils::getContextCopyWithTaskSettings(getContext().context, manifest); EXPECT_EQ( - worker_context->getSettingsRef()[Setting::export_merge_tree_part_schema_mismatch_mode].value, - MergeTreePartExportSchemaMismatchMode::strict); + worker_context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value, + MergeTreePartExportSchemaMatchMode::match_by_position); + EXPECT_EQ( + worker_context->getSettingsRef()[Setting::ignore_extra_source_columns].value, + false); } -TEST_F(ExportPartitionManifestBackCompatTest, SchemaMismatchModeAppliedToWorkerContextForEveryValue) +TEST_F(ExportPartitionManifestBackCompatTest, SchemaMatchModeAppliedToWorkerContextForEveryValue) { - for (const auto value : magic_enum::enum_values()) + for (const auto value : magic_enum::enum_values()) { auto manifest = makeValidManifest(); - manifest.schema_mismatch_mode = value; + manifest.schema_match_mode = value; auto worker_context = ExportPartitionUtils::getContextCopyWithTaskSettings(getContext().context, manifest); EXPECT_EQ( - worker_context->getSettingsRef()[Setting::export_merge_tree_part_schema_mismatch_mode].value, + worker_context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value, value) << "value=" << magic_enum::enum_name(value); } } +TEST_F(ExportPartitionManifestBackCompatTest, IgnoreExtraSourceColumnsAppliedToWorkerContextForEveryValue) +{ + for (const bool value : {false, true}) + { + auto manifest = makeValidManifest(); + manifest.ignore_extra_source_columns = value; + + auto worker_context = ExportPartitionUtils::getContextCopyWithTaskSettings(getContext().context, manifest); + + EXPECT_EQ( + worker_context->getSettingsRef()[Setting::ignore_extra_source_columns].value, + value) << "value=" << value; + } +} + TEST(ExportColumnCastsTest, UsesSelectedMatchingMode) { const ColumnsWithTypeAndName source_columns = { @@ -222,7 +273,8 @@ TEST(ExportColumnCastsTest, UsesSelectedMatchingMode) ExportPartitionUtils::verifyExportColumnCastsAreSafe( source_columns, destination_columns, - MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position, + MergeTreePartExportSchemaMatchMode::match_by_position, + /*ignore_extra_source_columns=*/ true, destination_storage_id); }, ErrorCodes::INCOMPATIBLE_COLUMNS); @@ -230,11 +282,12 @@ TEST(ExportColumnCastsTest, UsesSelectedMatchingMode) EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( source_columns, destination_columns, - MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name, + MergeTreePartExportSchemaMatchMode::match_by_name, + /*ignore_extra_source_columns=*/ true, destination_storage_id)); } -TEST(ExportColumnCastsTest, MatchingByNameNoOpWithoutExtraSourceColumns) +TEST(ExportColumnCastsTest, MatchByNameAcceptsReorderedColumnsWithEqualColumnCount) { const ColumnsWithTypeAndName source_columns = { makeColumn("id"), @@ -248,18 +301,26 @@ TEST(ExportColumnCastsTest, MatchingByNameNoOpWithoutExtraSourceColumns) }; const StorageID destination_storage_id{"test", "destination"}; - for (const auto mode : - {MergeTreePartExportSchemaMismatchMode::strict, - MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position, - MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name}) + for (const bool ignore_extra_source_columns : {false, true}) { expectExceptionCode( [&] { ExportPartitionUtils::verifyExportColumnCastsAreSafe( - source_columns, reordered_destination_columns, mode, destination_storage_id); + source_columns, + reordered_destination_columns, + MergeTreePartExportSchemaMatchMode::match_by_position, + ignore_extra_source_columns, + destination_storage_id); }, ErrorCodes::INCOMPATIBLE_COLUMNS); + + EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( + source_columns, + reordered_destination_columns, + MergeTreePartExportSchemaMatchMode::match_by_name, + ignore_extra_source_columns, + destination_storage_id)); } const ColumnsWithTypeAndName same_order_destination_columns = { @@ -268,14 +329,43 @@ TEST(ExportColumnCastsTest, MatchingByNameNoOpWithoutExtraSourceColumns) makeColumn("payload"), }; - for (const auto mode : - {MergeTreePartExportSchemaMismatchMode::strict, - MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position, - MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name}) - { - EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( - source_columns, same_order_destination_columns, mode, destination_storage_id)); - } + for (const auto mode : {MergeTreePartExportSchemaMatchMode::match_by_position, MergeTreePartExportSchemaMatchMode::match_by_name}) + for (const bool ignore_extra_source_columns : {false, true}) + EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( + source_columns, same_order_destination_columns, mode, ignore_extra_source_columns, destination_storage_id)); +} + +TEST(ExportColumnCastsTest, MatchByNameRejectsUnmatchedSourceColumnUnlessIgnored) +{ + const ColumnsWithTypeAndName source_columns = { + makeColumn("id"), + makeColumn("year"), + makeColumn("extra"), + }; + const ColumnsWithTypeAndName destination_columns = { + makeColumn("id"), + makeColumn("year"), + }; + const StorageID destination_storage_id{"test", "destination"}; + + expectExceptionCode( + [&] + { + ExportPartitionUtils::verifyExportColumnCastsAreSafe( + source_columns, + destination_columns, + MergeTreePartExportSchemaMatchMode::match_by_name, + /*ignore_extra_source_columns=*/ false, + destination_storage_id); + }, + ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH); + + EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( + source_columns, + destination_columns, + MergeTreePartExportSchemaMatchMode::match_by_name, + /*ignore_extra_source_columns=*/ true, + destination_storage_id)); } TEST(ExportColumnCastsTest, RejectsLossyCastAfterMatchingByName) @@ -296,7 +386,8 @@ TEST(ExportColumnCastsTest, RejectsLossyCastAfterMatchingByName) ExportPartitionUtils::verifyExportColumnCastsAreSafe( source_columns, destination_columns, - MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name, + MergeTreePartExportSchemaMatchMode::match_by_name, + /*ignore_extra_source_columns=*/ true, StorageID{"test", "destination"}); }, ErrorCodes::INCOMPATIBLE_COLUMNS); @@ -320,7 +411,8 @@ TEST(ExportColumnCastsTest, RejectsMissingDestinationColumnAfterMatchingByName) ExportPartitionUtils::verifyExportColumnCastsAreSafe( source_columns, destination_columns, - MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name, + MergeTreePartExportSchemaMatchMode::match_by_name, + /*ignore_extra_source_columns=*/ true, StorageID{"test", "destination"}); }, ErrorCodes::THERE_IS_NO_COLUMN); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index ed4a82c69d7f..b3bd2e8db53f 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -229,7 +229,8 @@ namespace Setting extern const SettingsBool export_merge_tree_part_throw_on_pending_mutations; extern const SettingsBool export_merge_tree_part_throw_on_pending_patch_parts; extern const SettingsBool export_merge_tree_part_allow_lossy_cast; - extern const SettingsMergeTreePartExportSchemaMismatchMode export_merge_tree_part_schema_mismatch_mode; + extern const SettingsMergeTreePartExportSchemaMatchMode export_merge_tree_part_schema_match_mode; + extern const SettingsBool ignore_extra_source_columns; extern const SettingsExportPartitionAllOnError export_merge_tree_partition_all_on_error; extern const SettingsString export_merge_tree_part_filename_pattern; extern const SettingsBool write_full_path_in_iceberg_metadata; @@ -8535,7 +8536,8 @@ void StorageReplicatedMergeTree::exportPartitionToTable(const PartitionCommand & manifest.filename_pattern = query_context->getSettingsRef()[Setting::export_merge_tree_part_filename_pattern].value; manifest.write_full_path_in_iceberg_metadata = query_context->getSettingsRef()[Setting::write_full_path_in_iceberg_metadata]; manifest.allow_lossy_cast = query_context->getSettingsRef()[Setting::export_merge_tree_part_allow_lossy_cast]; - manifest.schema_mismatch_mode = query_context->getSettingsRef()[Setting::export_merge_tree_part_schema_mismatch_mode].value; + manifest.schema_match_mode = query_context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value; + manifest.ignore_extra_source_columns = query_context->getSettingsRef()[Setting::ignore_extra_source_columns].value; if (dest_storage->isDataLake()) { diff --git a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py index 157832f56547..578592f6ee0f 100644 --- a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py +++ b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py @@ -38,8 +38,8 @@ EXTRA_SOURCE_COLUMN_MODES = [ - pytest.param("ignore_extra_source_columns_by_position", id="by-position"), - pytest.param("ignore_extra_source_columns_by_name", id="by-name"), + pytest.param("match_by_position", id="by-position"), + pytest.param("match_by_name", id="by-name"), ] @@ -763,8 +763,8 @@ def test_export_part_column_count_mismatch_source_fewer_is_rejected(cluster): node.query(f"DROP TABLE IF EXISTS {iceberg}") -@pytest.mark.parametrize("mismatch_mode", EXTRA_SOURCE_COLUMN_MODES) -def test_export_part_source_more_columns_allowed_with_ignore_extra_setting(cluster, mismatch_mode): +@pytest.mark.parametrize("schema_match_mode", EXTRA_SOURCE_COLUMN_MODES) +def test_export_part_source_more_columns_allowed_with_ignore_extra_setting(cluster, schema_match_mode): node = cluster.instances["node1"] sfx = unique_suffix() mt = f"mt_ignore_extra_{sfx}" @@ -778,7 +778,10 @@ def test_export_part_source_more_columns_allowed_with_ignore_extra_setting(clust export_part( node=node, table=mt, part=part_2020, dest=iceberg, - extra_settings=f"export_merge_tree_part_schema_mismatch_mode = '{mismatch_mode}'", + extra_settings=( + f"export_merge_tree_part_schema_match_mode = '{schema_match_mode}', " + f"ignore_extra_source_columns = 1" + ), ) wait_for_export_part(node=node, table=mt, part=part_2020) @@ -795,14 +798,14 @@ def test_export_part_source_more_columns_allowed_with_ignore_extra_setting(clust @pytest.mark.parametrize( - "mismatch_mode,expected_error", + "schema_match_mode,expected_error", [ - pytest.param("ignore_extra_source_columns_by_position", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-position"), - pytest.param("ignore_extra_source_columns_by_name", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-name"), + pytest.param("match_by_position", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-position"), + pytest.param("match_by_name", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-name"), ], ) def test_export_part_column_count_mismatch_source_fewer_still_rejected_with_ignore_extra_setting( - cluster, mismatch_mode, expected_error + cluster, schema_match_mode, expected_error ): node = cluster.instances["node1"] sfx = unique_suffix() @@ -819,9 +822,10 @@ def test_export_part_column_count_mismatch_source_fewer_still_rejected_with_igno f"ALTER TABLE {mt} EXPORT PART '{part_2020}' TO TABLE {iceberg} " f"SETTINGS allow_experimental_export_merge_tree_part = 1, " f"allow_experimental_insert_into_iceberg = 1, " - f"export_merge_tree_part_schema_mismatch_mode = '{mismatch_mode}'" + f"export_merge_tree_part_schema_match_mode = '{schema_match_mode}', " + f"ignore_extra_source_columns = 1" ) - assert expected_error in error, f"Expected {expected_error} with {mismatch_mode}, got: {error!r}" + assert expected_error in error, f"Expected {expected_error} with {schema_match_mode}, got: {error!r}" node.query(f"DROP TABLE IF EXISTS {mt} SYNC") node.query(f"DROP TABLE IF EXISTS {iceberg}") @@ -873,7 +877,7 @@ def test_export_part_ignore_extra_column_breaks_hybrid_over_source_and_destinati export_part( node=node, table=mt, part=part, dest=iceberg, - extra_settings="export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_position'", + extra_settings="export_merge_tree_part_schema_match_mode = 'match_by_position', ignore_extra_source_columns = 1", ) wait_for_export_part(node=node, table=mt, part=part) @@ -951,8 +955,8 @@ def test_export_part_with_alias_column(cluster): node.query(f"DROP TABLE IF EXISTS {iceberg}") -@pytest.mark.parametrize("mismatch_mode", EXTRA_SOURCE_COLUMN_MODES) -def test_export_part_ignore_extra_setting_drops_trailing_alias_column(cluster, mismatch_mode): +@pytest.mark.parametrize("schema_match_mode", EXTRA_SOURCE_COLUMN_MODES) +def test_export_part_ignore_extra_setting_drops_trailing_alias_column(cluster, schema_match_mode): node = cluster.instances["node1"] sfx = unique_suffix() mt = f"mt_ignore_extra_alias_{sfx}" @@ -966,7 +970,10 @@ def test_export_part_ignore_extra_setting_drops_trailing_alias_column(cluster, m export_part( node=node, table=mt, part=part_2020, dest=iceberg, - extra_settings=f"export_merge_tree_part_schema_mismatch_mode = '{mismatch_mode}'", + extra_settings=( + f"export_merge_tree_part_schema_match_mode = '{schema_match_mode}', " + f"ignore_extra_source_columns = 1" + ), ) wait_for_export_part(node=node, table=mt, part=part_2020) @@ -979,8 +986,8 @@ def test_export_part_ignore_extra_setting_drops_trailing_alias_column(cluster, m node.query(f"DROP TABLE IF EXISTS {iceberg}") -@pytest.mark.parametrize("mismatch_mode", EXTRA_SOURCE_COLUMN_MODES) -def test_export_part_ignore_extra_setting_kept_alias_depends_on_dropped_column(cluster, mismatch_mode): +@pytest.mark.parametrize("schema_match_mode", EXTRA_SOURCE_COLUMN_MODES) +def test_export_part_ignore_extra_setting_kept_alias_depends_on_dropped_column(cluster, schema_match_mode): node = cluster.instances["node1"] sfx = unique_suffix() mt = f"mt_ignore_extra_dep_{sfx}" @@ -998,7 +1005,10 @@ def test_export_part_ignore_extra_setting_kept_alias_depends_on_dropped_column(c export_part( node=node, table=mt, part=part_2020, dest=iceberg, - extra_settings=f"export_merge_tree_part_schema_mismatch_mode = '{mismatch_mode}'", + extra_settings=( + f"export_merge_tree_part_schema_match_mode = '{schema_match_mode}', " + f"ignore_extra_source_columns = 1" + ), ) wait_for_export_part(node=node, table=mt, part=part_2020) @@ -1072,7 +1082,8 @@ def test_export_part_reordered_subset_requires_matching_by_name(cluster): f"ALTER TABLE {mt} EXPORT PART '{part_2020}' TO TABLE {iceberg} " f"SETTINGS allow_experimental_export_merge_tree_part = 1, " f"allow_experimental_insert_into_iceberg = 1, " - f"export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_position'" + f"export_merge_tree_part_schema_match_mode = 'match_by_position', " + f"ignore_extra_source_columns = 1" ) assert "INCOMPATIBLE_COLUMNS" in error, f"Expected positional matching to fail, got: {error!r}" assert node.query(f"SELECT count() FROM {iceberg}").strip() == "0" @@ -1082,7 +1093,7 @@ def test_export_part_reordered_subset_requires_matching_by_name(cluster): mt, part_2020, iceberg, - extra_settings="export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_name'", + extra_settings="export_merge_tree_part_schema_match_mode = 'match_by_name', ignore_extra_source_columns = 1", ) wait_for_export_part(node, mt, part_2020) @@ -1117,7 +1128,8 @@ def test_export_part_match_by_name_requires_every_destination_column(cluster): f"ALTER TABLE {mt} EXPORT PART '{part_2020}' TO TABLE {iceberg} " f"SETTINGS allow_experimental_export_merge_tree_part = 1, " f"allow_experimental_insert_into_iceberg = 1, " - f"export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_name'" + f"export_merge_tree_part_schema_match_mode = 'match_by_name', " + f"ignore_extra_source_columns = 1" ) assert "THERE_IS_NO_COLUMN" in error and "renamed_id" in error, ( f"Expected name matching to reject missing column `renamed_id`, got: {error!r}" @@ -1129,8 +1141,10 @@ def test_export_part_match_by_name_requires_every_destination_column(cluster): node.query(f"DROP TABLE IF EXISTS {iceberg}") -def test_export_part_match_by_name_falls_back_to_positional_without_extra_source_columns(cluster): - # No extra source column here, so `ignore_extra_source_columns_by_name` behaves like `strict`. +def test_export_part_match_by_name_rejects_renamed_column_without_extra_source_columns(cluster): + """`match_by_name` has no positional fallback: even when the source and destination have the + exact same number of columns (no extra source column to justify falling back to position), + a destination column absent from the source by name must still be rejected.""" node = cluster.instances["node1"] sfx = unique_suffix() mt = f"mt_match_by_name_fallback_{sfx}" @@ -1142,14 +1156,48 @@ def test_export_part_match_by_name_falls_back_to_positional_without_extra_source node.query(f"INSERT INTO {mt} VALUES (1, 2020), (2, 2020)") part_2020 = get_part(node, mt, "2020") + error = node.query_and_get_error( + f"ALTER TABLE {mt} EXPORT PART '{part_2020}' TO TABLE {iceberg} " + f"SETTINGS allow_experimental_export_merge_tree_part = 1, " + f"allow_experimental_insert_into_iceberg = 1, " + f"export_merge_tree_part_schema_match_mode = 'match_by_name'" + ) + assert "THERE_IS_NO_COLUMN" in error and "renamed_id" in error, ( + f"Expected name matching to reject missing column `renamed_id` even without an extra " + f"source column (no positional fallback), got: {error!r}" + ) + assert node.query(f"SELECT count() FROM {iceberg}").strip() == "0" + + node.query(f"DROP TABLE IF EXISTS {mt} SYNC") + node.query(f"DROP TABLE IF EXISTS {iceberg}") + + +def test_export_part_match_by_name_reorders_columns_without_extra_source_columns(cluster): + """The scenario `export_merge_tree_part_schema_match_mode = 'match_by_name'` was introduced for: + the source and destination have the exact same number of columns, declared in a different order. + Previously this fell back to positional matching (a no-op for by-name mode); now it is matched + by name like any other case.""" + node = cluster.instances["node1"] + sfx = unique_suffix() + mt = f"mt_match_by_name_equal_count_{sfx}" + iceberg = f"iceberg_match_by_name_equal_count_{sfx}" + + make_mt(node, mt, "id Int32, year Int32, payload String", "year") + make_iceberg_s3(node, iceberg, "payload String, id Int32, year Int32", "year") + + node.query(f"INSERT INTO {mt} VALUES (1, 2020, 'foo'), (2, 2020, 'bar')") + part_2020 = get_part(node, mt, "2020") + export_part( node, mt, part_2020, iceberg, - extra_settings="export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_name'", + extra_settings="export_merge_tree_part_schema_match_mode = 'match_by_name'", ) wait_for_export_part(node, mt, part_2020) - result = node.query(f"SELECT renamed_id, year FROM {iceberg} ORDER BY renamed_id").strip() - assert result == "1\t2020\n2\t2020", f"Unexpected data:\n{result}" + result = node.query(f"SELECT id, year, payload FROM {iceberg} ORDER BY id").strip() + assert result == "1\t2020\tfoo\n2\t2020\tbar", f"Unexpected data:\n{result}" + + assert_part_log(node, mt, part_2020) node.query(f"DROP TABLE IF EXISTS {mt} SYNC") node.query(f"DROP TABLE IF EXISTS {iceberg}") diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index 24fe43c2806f..72e0ccc11217 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -10,8 +10,8 @@ EXTRA_SOURCE_COLUMN_MODES = [ - pytest.param("ignore_extra_source_columns_by_position", id="by-position"), - pytest.param("ignore_extra_source_columns_by_name", id="by-name"), + pytest.param("match_by_position", id="by-position"), + pytest.param("match_by_name", id="by-name"), ] @@ -1037,14 +1037,14 @@ def test_export_part_column_count_mismatch_source_fewer_is_rejected(cluster): @pytest.mark.parametrize( - "mismatch_mode,expected_error", + "schema_match_mode,expected_error", [ - pytest.param("ignore_extra_source_columns_by_position", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-position"), - pytest.param("ignore_extra_source_columns_by_name", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-name"), + pytest.param("match_by_position", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-position"), + pytest.param("match_by_name", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-name"), ], ) def test_export_part_column_count_mismatch_source_fewer_still_rejected_with_ignore_extra_setting( - cluster, mismatch_mode, expected_error + cluster, schema_match_mode, expected_error ): node = cluster.instances["node1"] @@ -1067,10 +1067,11 @@ def test_export_part_column_count_mismatch_source_fewer_still_rejected_with_igno error = node.query_and_get_error( f"ALTER TABLE {mt_table} EXPORT PART '2020_1_1_0' TO TABLE {s3_table} " - f"SETTINGS export_merge_tree_part_schema_mismatch_mode = '{mismatch_mode}'" + f"SETTINGS export_merge_tree_part_schema_match_mode = '{schema_match_mode}', " + f"ignore_extra_source_columns = 1" ) assert expected_error in error, ( - f"Expected {expected_error} for source Date: Mon, 24 Aug 2026 12:46:32 +0200 Subject: [PATCH 07/11] some updates Signed-off-by: Konstantin Morozov --- .../MergeTree/ExportPartitionUtils.cpp | 138 +++++++++--------- src/Storages/MergeTree/ExportPartitionUtils.h | 1 - .../tests/gtest_export_partition_ordering.cpp | 59 +++----- 3 files changed, 83 insertions(+), 115 deletions(-) diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index d19bb423c6c7..1be0509b9b1c 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -779,51 +779,34 @@ namespace ExportPartitionUtils const ColumnsWithTypeAndName & source_columns, const ColumnsWithTypeAndName & destination_columns, MergeTreePartExportSchemaMatchMode schema_match_mode, - bool ignore_extra_source_columns, const StorageID & destination_storage_id) { - if (schema_match_mode == MergeTreePartExportSchemaMatchMode::match_by_name) + switch (schema_match_mode) { - /// `ignore_extra_source_columns` only excuses the source having MORE columns than the - /// destination; a source with fewer columns is always a mismatch. - const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); - if (source_columns.size() != destination_columns.size() - && !(ignore_extra_source_columns && src_has_extra_columns)) - throw Exception( - ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, - "Number of columns doesn't match (source: {} and result: {})", - source_columns.size(), - destination_columns.size()); + case MergeTreePartExportSchemaMatchMode::match_by_position: { + for (size_t i = 0; i < destination_columns.size(); ++i) + verifyExportColumnCastIsSafe(source_columns[i], destination_columns[i], destination_storage_id); + return; + } + case MergeTreePartExportSchemaMatchMode::match_by_name: { + std::unordered_map source_columns_by_name; + source_columns_by_name.reserve(source_columns.size()); + for (const auto & source_column : source_columns) + source_columns_by_name.emplace(source_column.name, &source_column); - std::unordered_map source_columns_by_name; - source_columns_by_name.reserve(source_columns.size()); - for (const auto & source_column : source_columns) - source_columns_by_name.emplace(source_column.name, &source_column); + for (const auto & destination_column : destination_columns) + { + const auto source_it = source_columns_by_name.find(destination_column.name); + if (source_it == source_columns_by_name.end()) + throw Exception( + ErrorCodes::THERE_IS_NO_COLUMN, "Cannot find column `{}` in source stream", destination_column.name); - for (const auto & destination_column : destination_columns) - { - const auto source_it = source_columns_by_name.find(destination_column.name); - if (source_it == source_columns_by_name.end()) - throw Exception( - ErrorCodes::THERE_IS_NO_COLUMN, - "Cannot find column `{}` in source stream", - destination_column.name); - - verifyExportColumnCastIsSafe(*source_it->second, destination_column, destination_storage_id); + verifyExportColumnCastIsSafe(*source_it->second, destination_column, destination_storage_id); + } + return; } - return; } - - if (source_columns.size() < destination_columns.size() - || (!ignore_extra_source_columns && source_columns.size() != destination_columns.size())) - throw Exception( - ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, - "Number of columns doesn't match (source: {} and result: {})", - source_columns.size(), - destination_columns.size()); - - for (size_t i = 0; i < destination_columns.size(); ++i) - verifyExportColumnCastIsSafe(source_columns[i], destination_columns[i], destination_storage_id); + UNREACHABLE(); } void verifyExportSchemaCastable( @@ -850,6 +833,22 @@ namespace ExportPartitionUtils const bool match_by_name = schema_match_mode == MergeTreePartExportSchemaMatchMode::match_by_name; const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); + if (source_columns.size() < destination_columns.size()) + throw Exception( + ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, + "Number of columns doesn't match (source: {} and result: {}): " + "destination cannot have more columns than source", + source_columns.size(), + destination_columns.size()); + + if (src_has_extra_columns && !ignore_extra_source_columns) + throw Exception( + ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, + "Number of columns doesn't match (source: {} and result: {}): " + "source has extra columns and the `ignore_extra_source_columns` setting is disabled", + source_columns.size(), + destination_columns.size()); + if (!match_by_name && ignore_extra_source_columns && src_has_extra_columns) { LOG_DEBUG(getLogger("ExportPartitionUtils"), @@ -861,17 +860,6 @@ namespace ExportPartitionUtils source_columns.resize(destination_columns.size()); } - /// `makeConvertingActions` in `Name` mode silently ignores an unreferenced source column, so it must be rejected here explicitly. - /// `ignore_extra_source_columns` only excuses the source having MORE columns than the destination; - /// a source with fewer columns is always a mismatch. - if (match_by_name && source_columns.size() != destination_columns.size() - && !(ignore_extra_source_columns && src_has_extra_columns)) - throw Exception( - ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, - "Number of columns doesn't match (source: {} and result: {})", - source_columns.size(), - destination_columns.size()); - (void) ActionsDAG::makeConvertingActions( source_columns, destination_columns, @@ -894,34 +882,39 @@ namespace ExportPartitionUtils const bool allow_lossy_cast = context->getSettingsRef()[Setting::export_merge_tree_part_allow_lossy_cast]; - if (match_by_name) + switch (schema_match_mode) { - std::unordered_map source_positions_by_name; - source_positions_by_name.reserve(source_columns.size()); - for (size_t i = 0; i < source_columns.size(); ++i) - source_positions_by_name.emplace(source_columns[i].name, i); - - for (const auto & destination_column : destination_columns) + case MergeTreePartExportSchemaMatchMode::match_by_name: { - if (!partition_key_owner_columns.contains(destination_column.name)) - continue; + std::unordered_map source_positions_by_name; + source_positions_by_name.reserve(source_columns.size()); + for (size_t i = 0; i < source_columns.size(); ++i) + source_positions_by_name.emplace(source_columns[i].name, i); - const auto source_it = source_positions_by_name.find(destination_column.name); - if (source_it == source_positions_by_name.end()) - continue; + for (const auto & destination_column : destination_columns) + { + if (!partition_key_owner_columns.contains(destination_column.name)) + continue; - verifyPartitionKeyColumn( - source_columns[source_it->second], destination_column, source_it->second, destination_storage_id, - /*match_by_name=*/ true); + const auto source_it = source_positions_by_name.find(destination_column.name); + if (source_it == source_positions_by_name.end()) + continue; + + verifyPartitionKeyColumn( + source_columns[source_it->second], destination_column, source_it->second, destination_storage_id, + /*match_by_name=*/ true); + } + break; + } + case MergeTreePartExportSchemaMatchMode::match_by_position: + { + const size_t num_columns = std::min(source_columns.size(), destination_columns.size()); + for (size_t i = 0; i < num_columns; ++i) + if (partition_key_owner_columns.contains(source_columns[i].name)) + verifyPartitionKeyColumn(source_columns[i], destination_columns[i], i, destination_storage_id, + /*match_by_name=*/ false); + break; } - } - else - { - const size_t num_columns = std::min(source_columns.size(), destination_columns.size()); - for (size_t i = 0; i < num_columns; ++i) - if (partition_key_owner_columns.contains(source_columns[i].name)) - verifyPartitionKeyColumn(source_columns[i], destination_columns[i], i, destination_storage_id, - /*match_by_name=*/ false); } /// Lossy casts may silently change values, so reject them unless the user opts in. @@ -932,7 +925,6 @@ namespace ExportPartitionUtils source_columns, destination_columns, schema_match_mode, - ignore_extra_source_columns, destination_storage_id); } } diff --git a/src/Storages/MergeTree/ExportPartitionUtils.h b/src/Storages/MergeTree/ExportPartitionUtils.h index a56377728b79..6e9cdbab1b8b 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.h +++ b/src/Storages/MergeTree/ExportPartitionUtils.h @@ -99,7 +99,6 @@ namespace ExportPartitionUtils const ColumnsWithTypeAndName & source_columns, const ColumnsWithTypeAndName & destination_columns, MergeTreePartExportSchemaMatchMode schema_match_mode, - bool ignore_extra_source_columns, const StorageID & destination_storage_id); /// Validates that source columns can be exported into the destination with the configured diff --git a/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp b/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp index 71206642da1a..0a3be2fadad5 100644 --- a/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp +++ b/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp @@ -274,7 +274,6 @@ TEST(ExportColumnCastsTest, UsesSelectedMatchingMode) source_columns, destination_columns, MergeTreePartExportSchemaMatchMode::match_by_position, - /*ignore_extra_source_columns=*/ true, destination_storage_id); }, ErrorCodes::INCOMPATIBLE_COLUMNS); @@ -283,7 +282,6 @@ TEST(ExportColumnCastsTest, UsesSelectedMatchingMode) source_columns, destination_columns, MergeTreePartExportSchemaMatchMode::match_by_name, - /*ignore_extra_source_columns=*/ true, destination_storage_id)); } @@ -301,27 +299,22 @@ TEST(ExportColumnCastsTest, MatchByNameAcceptsReorderedColumnsWithEqualColumnCou }; const StorageID destination_storage_id{"test", "destination"}; - for (const bool ignore_extra_source_columns : {false, true}) - { - expectExceptionCode( - [&] - { - ExportPartitionUtils::verifyExportColumnCastsAreSafe( - source_columns, - reordered_destination_columns, - MergeTreePartExportSchemaMatchMode::match_by_position, - ignore_extra_source_columns, - destination_storage_id); - }, - ErrorCodes::INCOMPATIBLE_COLUMNS); + expectExceptionCode( + [&] + { + ExportPartitionUtils::verifyExportColumnCastsAreSafe( + source_columns, + reordered_destination_columns, + MergeTreePartExportSchemaMatchMode::match_by_position, + destination_storage_id); + }, + ErrorCodes::INCOMPATIBLE_COLUMNS); - EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( - source_columns, - reordered_destination_columns, - MergeTreePartExportSchemaMatchMode::match_by_name, - ignore_extra_source_columns, - destination_storage_id)); - } + EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( + source_columns, + reordered_destination_columns, + MergeTreePartExportSchemaMatchMode::match_by_name, + destination_storage_id)); const ColumnsWithTypeAndName same_order_destination_columns = { makeColumn("id"), @@ -330,12 +323,11 @@ TEST(ExportColumnCastsTest, MatchByNameAcceptsReorderedColumnsWithEqualColumnCou }; for (const auto mode : {MergeTreePartExportSchemaMatchMode::match_by_position, MergeTreePartExportSchemaMatchMode::match_by_name}) - for (const bool ignore_extra_source_columns : {false, true}) - EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( - source_columns, same_order_destination_columns, mode, ignore_extra_source_columns, destination_storage_id)); + EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( + source_columns, same_order_destination_columns, mode, destination_storage_id)); } -TEST(ExportColumnCastsTest, MatchByNameRejectsUnmatchedSourceColumnUnlessIgnored) +TEST(ExportColumnCastsTest, MatchByNameToleratesUnmatchedSourceColumn) { const ColumnsWithTypeAndName source_columns = { makeColumn("id"), @@ -348,23 +340,10 @@ TEST(ExportColumnCastsTest, MatchByNameRejectsUnmatchedSourceColumnUnlessIgnored }; const StorageID destination_storage_id{"test", "destination"}; - expectExceptionCode( - [&] - { - ExportPartitionUtils::verifyExportColumnCastsAreSafe( - source_columns, - destination_columns, - MergeTreePartExportSchemaMatchMode::match_by_name, - /*ignore_extra_source_columns=*/ false, - destination_storage_id); - }, - ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH); - EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( source_columns, destination_columns, MergeTreePartExportSchemaMatchMode::match_by_name, - /*ignore_extra_source_columns=*/ true, destination_storage_id)); } @@ -387,7 +366,6 @@ TEST(ExportColumnCastsTest, RejectsLossyCastAfterMatchingByName) source_columns, destination_columns, MergeTreePartExportSchemaMatchMode::match_by_name, - /*ignore_extra_source_columns=*/ true, StorageID{"test", "destination"}); }, ErrorCodes::INCOMPATIBLE_COLUMNS); @@ -412,7 +390,6 @@ TEST(ExportColumnCastsTest, RejectsMissingDestinationColumnAfterMatchingByName) source_columns, destination_columns, MergeTreePartExportSchemaMatchMode::match_by_name, - /*ignore_extra_source_columns=*/ true, StorageID{"test", "destination"}); }, ErrorCodes::THERE_IS_NO_COLUMN); From 2b3e0af86c7e89646e7f226abf6f40d80946df54 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Mon, 24 Aug 2026 13:11:36 +0200 Subject: [PATCH 08/11] rename setting Signed-off-by: Konstantin Morozov --- docs/en/antalya/part_export.md | 12 +++++------ docs/en/antalya/partition_export.md | 12 +++++------ src/Core/Settings.cpp | 4 ++-- src/Core/SettingsChangesHistory.cpp | 2 +- src/Storages/MergeTree/ExportPartTask.cpp | 4 ++-- .../MergeTree/ExportPartitionUtils.cpp | 8 ++++---- src/Storages/MergeTree/ExportPartitionUtils.h | 2 +- .../tests/gtest_export_partition_ordering.cpp | 6 +++--- src/Storages/StorageReplicatedMergeTree.cpp | 4 ++-- .../test.py | 16 +++++++-------- .../test.py | 8 ++++---- .../test.py | 20 +++++++++---------- .../test.py | 4 ++-- 13 files changed, 51 insertions(+), 51 deletions(-) diff --git a/docs/en/antalya/part_export.md b/docs/en/antalya/part_export.md index 1637bf7d6a8d..fd62919a247b 100644 --- a/docs/en/antalya/part_export.md +++ b/docs/en/antalya/part_export.md @@ -55,7 +55,7 @@ Source and destination tables must support positional schema conversion. The fol The following must match between source and destination: -1. **Column count** - by default (`ignore_extra_source_columns = false`) every source column must have a corresponding destination column: with `export_merge_tree_part_schema_match_mode = 'match_by_position'` (the default) the source and destination must have the same number of columns; with `'match_by_name'` they must have the same set of column names. A mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. Set `ignore_extra_source_columns = 1` to allow a source table with columns that have no corresponding destination column; such columns are dropped and not exported. The destination having a column absent from the source is always rejected, regardless of this setting. +1. **Column count** - by default (`export_merge_tree_part_ignore_extra_source_columns = false`) every source column must have a corresponding destination column: with `export_merge_tree_part_schema_match_mode = 'match_by_position'` (the default) the source and destination must have the same number of columns; with `'match_by_name'` they must have the same set of column names. A mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. Set `export_merge_tree_part_ignore_extra_source_columns = 1` to allow a source table with columns that have no corresponding destination column; such columns are dropped and not exported. The destination having a column absent from the source is always rejected, regardless of this setting. 2. **`PARTITION BY` expressions** - for destinations other than data lakes, the source and destination `PARTITION BY` expressions must be identical. For Apache Iceberg destinations, the source partition key must be representable as an Iceberg partition spec and must match the destination partition fields and transforms. 3. **The position of every column backing the partition key** - it is not enough for the `PARTITION BY` expressions to be textually identical: every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. If such a column contains a named `Tuple`, its element names must also be declared in the same order (an unnamed `Tuple` on either side is exempt from this, per the allowance above). This comparison is recursive through nested tuples and through container types such as `Array` and `Map`. @@ -144,9 +144,9 @@ In case a table function is used as the destination, the schema can be omitted a - `match_by_position` (default) - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. Column names are not otherwise considered. - `match_by_name` - every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be declared in a different order than the source. A destination column absent from the source, including when it was renamed, throws `THERE_IS_NO_COLUMN`; there is no positional fallback. - See `ignore_extra_source_columns` below for how a source column without a corresponding destination column is handled in each mode. + See `export_merge_tree_part_ignore_extra_source_columns` below for how a source column without a corresponding destination column is handled in each mode. -### `ignore_extra_source_columns` (Optional) +### `export_merge_tree_part_ignore_extra_source_columns` (Optional) - **Type**: `Bool` - **Default**: `false` @@ -158,9 +158,9 @@ In case a table function is used as the destination, the schema can be omitted a Error behavior: - - With `ignore_extra_source_columns = false` (default), a source column without a corresponding destination column throws `NUMBER_OF_COLUMNS_DOESNT_MATCH` - in `match_by_position` mode this means any column-count mismatch, in `match_by_name` mode this means the source and destination column-name sets differ. - - In `match_by_position` mode, the destination having more columns than the source always throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`, regardless of `ignore_extra_source_columns`. - - In `match_by_name` mode, a destination column absent from the source (including a renamed one) always throws `THERE_IS_NO_COLUMN`, regardless of `ignore_extra_source_columns`; there is no positional fallback. + - With `export_merge_tree_part_ignore_extra_source_columns = false` (default), a source column without a corresponding destination column throws `NUMBER_OF_COLUMNS_DOESNT_MATCH` - in `match_by_position` mode this means any column-count mismatch, in `match_by_name` mode this means the source and destination column-name sets differ. + - In `match_by_position` mode, the destination having more columns than the source always throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`, regardless of `export_merge_tree_part_ignore_extra_source_columns`. + - In `match_by_name` mode, a destination column absent from the source (including a renamed one) always throws `THERE_IS_NO_COLUMN`, regardless of `export_merge_tree_part_ignore_extra_source_columns`; there is no positional fallback. - After columns have been matched successfully, a cast rejected by the export type-safety check throws `INCOMPATIBLE_COLUMNS`. diff --git a/docs/en/antalya/partition_export.md b/docs/en/antalya/partition_export.md index 17b42b91f66c..ff8c47c7a110 100644 --- a/docs/en/antalya/partition_export.md +++ b/docs/en/antalya/partition_export.md @@ -47,7 +47,7 @@ TO TABLE [destination_database.]destination_table `EXPORT PARTITION` exports each part via the same mechanism as [`EXPORT PART`](/docs/en/antalya/part_export.md#requirements), so the source and destination tables must satisfy the same compatibility requirements. Columns are matched by position by default, or by their exact, case-sensitive name if `export_merge_tree_part_schema_match_mode = 'match_by_name'` is set, and column types may differ as long as they are safely castable (or `export_merge_tree_part_allow_lossy_cast = 1` is set). Beyond that, the following must match: -1. **Column count** - by default (`ignore_extra_source_columns = false`) every source column must have a corresponding destination column: with `export_merge_tree_part_schema_match_mode = 'match_by_position'` (the default) the source and destination must have the same number of columns; with `'match_by_name'` they must have the same set of column names. Set `ignore_extra_source_columns = 1` to allow a source table with columns that have no corresponding destination column; the destination having a column absent from the source is still always rejected. +1. **Column count** - by default (`export_merge_tree_part_ignore_extra_source_columns = false`) every source column must have a corresponding destination column: with `export_merge_tree_part_schema_match_mode = 'match_by_position'` (the default) the source and destination must have the same number of columns; with `'match_by_name'` they must have the same set of column names. Set `export_merge_tree_part_ignore_extra_source_columns = 1` to allow a source table with columns that have no corresponding destination column; the destination having a column absent from the source is still always rejected. 2. **`PARTITION BY` expressions** - for destinations other than data lakes, the source and destination `PARTITION BY` expressions must be identical. For Apache Iceberg destinations, the source partition key must match the destination partition fields and transforms. 3. **Partition key column positions and layouts** - every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. Named `Tuple` elements within such a column must also be declared in the same order, including tuples nested inside `Array` or `Map`. This applies even if both tables' `PARTITION BY` expressions are textually identical. See [`EXPORT PART` requirements](/docs/en/antalya/part_export.md#requirements) for a worked example and the corresponding exception message. @@ -136,9 +136,9 @@ Notes: - `match_by_position` (default) - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. Column names are not otherwise considered. - `match_by_name` - every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be declared in a different order than the source. A destination column absent from the source, including when it was renamed, throws `THERE_IS_NO_COLUMN`; there is no positional fallback. - See `ignore_extra_source_columns` below for how a source column without a corresponding destination column is handled in each mode. + See `export_merge_tree_part_ignore_extra_source_columns` below for how a source column without a corresponding destination column is handled in each mode. -### `ignore_extra_source_columns` (Optional) +### `export_merge_tree_part_ignore_extra_source_columns` (Optional) - **Type**: `Bool` - **Default**: `false` @@ -150,9 +150,9 @@ Notes: Error behavior: - - With `ignore_extra_source_columns = false` (default), a source column without a corresponding destination column throws `NUMBER_OF_COLUMNS_DOESNT_MATCH` - in `match_by_position` mode this means any column-count mismatch, in `match_by_name` mode this means the source and destination column-name sets differ. - - In `match_by_position` mode, the destination having more columns than the source always throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`, regardless of `ignore_extra_source_columns`. - - In `match_by_name` mode, a destination column absent from the source (including a renamed one) always throws `THERE_IS_NO_COLUMN`, regardless of `ignore_extra_source_columns`; there is no positional fallback. + - With `export_merge_tree_part_ignore_extra_source_columns = false` (default), a source column without a corresponding destination column throws `NUMBER_OF_COLUMNS_DOESNT_MATCH` - in `match_by_position` mode this means any column-count mismatch, in `match_by_name` mode this means the source and destination column-name sets differ. + - In `match_by_position` mode, the destination having more columns than the source always throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`, regardless of `export_merge_tree_part_ignore_extra_source_columns`. + - In `match_by_name` mode, a destination column absent from the source (including a renamed one) always throws `THERE_IS_NO_COLUMN`, regardless of `export_merge_tree_part_ignore_extra_source_columns`; there is no positional fallback. - After columns have been matched successfully, a cast rejected by the export type-safety check throws `INCOMPATIBLE_COLUMNS`. ## Examples diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 45eb39385754..ec9efc301dad 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -7633,9 +7633,9 @@ Possible values: - `match_by_position` (default) - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. Column names are not otherwise considered. - `match_by_name` - every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be declared in a different order than the source. A destination column absent from the source throws `THERE_IS_NO_COLUMN`, with no positional fallback. -See also `ignore_extra_source_columns`, which controls whether a source column without a corresponding destination column is dropped or rejected. +See also `export_merge_tree_part_ignore_extra_source_columns`, which controls whether a source column without a corresponding destination column is dropped or rejected. )", 0) \ - DECLARE(Bool, ignore_extra_source_columns, false, R"( + DECLARE(Bool, export_merge_tree_part_ignore_extra_source_columns, false, R"( Controls whether `EXPORT PART`/`EXPORT PARTITION` tolerates source `MergeTree` columns that have no corresponding destination column. - `false` (default) - such a source column is rejected: the source and destination must match exactly (in `export_merge_tree_part_schema_match_mode = 'match_by_position'`, this means the same number of columns; in `'match_by_name'`, the same set of column names). A mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - `true` - a source column without a corresponding destination column is dropped and not exported, instead of throwing. In `match_by_position` mode, this allows a source with extra trailing columns (the destination having more columns than the source is still always rejected). In `match_by_name` mode, this allows source columns whose name has no destination counterpart. diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 580d09fc8703..26df2661a4a7 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -45,7 +45,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"export_merge_tree_partition_task_timeout_seconds", "3600", "86400", "Increase default value to make it more realistic"}, {"export_merge_tree_part_allow_lossy_cast", false, false, "New setting to gate lossy casts in EXPORT PART/PARTITION behind explicit acknowledgment"}, {"export_merge_tree_part_schema_match_mode", "match_by_position", "match_by_position", "New setting to control how EXPORT PART/EXPORT PARTITION matches source columns to destination columns"}, - {"ignore_extra_source_columns", false, false, "New setting to allow EXPORT PART/EXPORT PARTITION when the source table has columns absent from the destination"}, + {"export_merge_tree_part_ignore_extra_source_columns", false, false, "New setting to allow EXPORT PART/EXPORT PARTITION when the source table has columns absent from the destination"}, {"export_merge_tree_partition_retry_initial_backoff_seconds", 5, 5, "New setting for exponential back-off between failed part export retries in an export partition task"}, {"export_merge_tree_partition_retry_max_backoff_seconds", 300, 300, "New setting capping the exponential back-off between failed part export retries in an export partition task"}, {"export_merge_tree_partition_max_retries", 3, 3, "Obsolete and ignored: export partition tasks now retry retryable failures until the task timeout and fail immediately on non-retryable errors, instead of using a fixed retry budget"}, diff --git a/src/Storages/MergeTree/ExportPartTask.cpp b/src/Storages/MergeTree/ExportPartTask.cpp index 2126601ce7c3..7cb2f051f2a5 100644 --- a/src/Storages/MergeTree/ExportPartTask.cpp +++ b/src/Storages/MergeTree/ExportPartTask.cpp @@ -66,7 +66,7 @@ namespace Setting extern const SettingsBool allow_experimental_analyzer; extern const SettingsString export_merge_tree_part_filename_pattern; extern const SettingsMergeTreePartExportSchemaMatchMode export_merge_tree_part_schema_match_mode; - extern const SettingsBool ignore_extra_source_columns; + extern const SettingsBool export_merge_tree_part_ignore_extra_source_columns; } namespace @@ -126,7 +126,7 @@ namespace const auto schema_match_mode = local_context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value; const bool ignore_extra_source_columns = - local_context->getSettingsRef()[Setting::ignore_extra_source_columns]; + local_context->getSettingsRef()[Setting::export_merge_tree_part_ignore_extra_source_columns]; const bool match_by_name = schema_match_mode == MergeTreePartExportSchemaMatchMode::match_by_name; auto source_columns = plan_for_part.getCurrentHeader()->getColumnsWithTypeAndName(); diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 1be0509b9b1c..544a7b9666da 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -85,7 +85,7 @@ namespace Setting { extern const SettingsBool export_merge_tree_part_allow_lossy_cast; extern const SettingsMergeTreePartExportSchemaMatchMode export_merge_tree_part_schema_match_mode; - extern const SettingsBool ignore_extra_source_columns; + extern const SettingsBool export_merge_tree_part_ignore_extra_source_columns; } namespace FailPoints @@ -184,7 +184,7 @@ namespace ExportPartitionUtils "export_merge_tree_part_schema_match_mode", String(magic_enum::enum_name(manifest.schema_match_mode.value_or(MergeTreePartExportSchemaMatchMode::match_by_position)))); context_copy->setSetting( - "ignore_extra_source_columns", + "export_merge_tree_part_ignore_extra_source_columns", manifest.ignore_extra_source_columns.value_or(false)); context_copy->setSetting("max_threads", manifest.max_threads); @@ -829,7 +829,7 @@ namespace ExportPartitionUtils const auto schema_match_mode = context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value; const bool ignore_extra_source_columns = - context->getSettingsRef()[Setting::ignore_extra_source_columns]; + context->getSettingsRef()[Setting::export_merge_tree_part_ignore_extra_source_columns]; const bool match_by_name = schema_match_mode == MergeTreePartExportSchemaMatchMode::match_by_name; const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); @@ -845,7 +845,7 @@ namespace ExportPartitionUtils throw Exception( ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, "Number of columns doesn't match (source: {} and result: {}): " - "source has extra columns and the `ignore_extra_source_columns` setting is disabled", + "source has extra columns and the `export_merge_tree_part_ignore_extra_source_columns` setting is disabled", source_columns.size(), destination_columns.size()); diff --git a/src/Storages/MergeTree/ExportPartitionUtils.h b/src/Storages/MergeTree/ExportPartitionUtils.h index 6e9cdbab1b8b..d0ba7563ac0d 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.h +++ b/src/Storages/MergeTree/ExportPartitionUtils.h @@ -103,7 +103,7 @@ namespace ExportPartitionUtils /// Validates that source columns can be exported into the destination with the configured /// positional or name-based CAST matching (`export_merge_tree_part_schema_match_mode`) and - /// unmatched-column policy (`ignore_extra_source_columns`). Lossy casts are rejected unless + /// unmatched-column policy (`export_merge_tree_part_ignore_extra_source_columns`). Lossy casts are rejected unless /// `export_merge_tree_part_allow_lossy_cast` is set. Throws BAD_ARGUMENTS on any violation. void verifyExportSchemaCastable( const StorageMetadataPtr & source_metadata, diff --git a/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp b/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp index 0a3be2fadad5..b01c564e63d9 100644 --- a/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp +++ b/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp @@ -24,7 +24,7 @@ namespace ErrorCodes namespace Setting { extern const SettingsMergeTreePartExportSchemaMatchMode export_merge_tree_part_schema_match_mode; - extern const SettingsBool ignore_extra_source_columns; + extern const SettingsBool export_merge_tree_part_ignore_extra_source_columns; } namespace @@ -218,7 +218,7 @@ TEST_F(ExportPartitionManifestBackCompatTest, MissingSchemaMatchSettingsFallBack worker_context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value, MergeTreePartExportSchemaMatchMode::match_by_position); EXPECT_EQ( - worker_context->getSettingsRef()[Setting::ignore_extra_source_columns].value, + worker_context->getSettingsRef()[Setting::export_merge_tree_part_ignore_extra_source_columns].value, false); } @@ -247,7 +247,7 @@ TEST_F(ExportPartitionManifestBackCompatTest, IgnoreExtraSourceColumnsAppliedToW auto worker_context = ExportPartitionUtils::getContextCopyWithTaskSettings(getContext().context, manifest); EXPECT_EQ( - worker_context->getSettingsRef()[Setting::ignore_extra_source_columns].value, + worker_context->getSettingsRef()[Setting::export_merge_tree_part_ignore_extra_source_columns].value, value) << "value=" << value; } } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index b3bd2e8db53f..609566e975b6 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -230,7 +230,7 @@ namespace Setting extern const SettingsBool export_merge_tree_part_throw_on_pending_patch_parts; extern const SettingsBool export_merge_tree_part_allow_lossy_cast; extern const SettingsMergeTreePartExportSchemaMatchMode export_merge_tree_part_schema_match_mode; - extern const SettingsBool ignore_extra_source_columns; + extern const SettingsBool export_merge_tree_part_ignore_extra_source_columns; extern const SettingsExportPartitionAllOnError export_merge_tree_partition_all_on_error; extern const SettingsString export_merge_tree_part_filename_pattern; extern const SettingsBool write_full_path_in_iceberg_metadata; @@ -8537,7 +8537,7 @@ void StorageReplicatedMergeTree::exportPartitionToTable(const PartitionCommand & manifest.write_full_path_in_iceberg_metadata = query_context->getSettingsRef()[Setting::write_full_path_in_iceberg_metadata]; manifest.allow_lossy_cast = query_context->getSettingsRef()[Setting::export_merge_tree_part_allow_lossy_cast]; manifest.schema_match_mode = query_context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value; - manifest.ignore_extra_source_columns = query_context->getSettingsRef()[Setting::ignore_extra_source_columns].value; + manifest.ignore_extra_source_columns = query_context->getSettingsRef()[Setting::export_merge_tree_part_ignore_extra_source_columns].value; if (dest_storage->isDataLake()) { diff --git a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py index 578592f6ee0f..05b6cfbd1d95 100644 --- a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py +++ b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py @@ -780,7 +780,7 @@ def test_export_part_source_more_columns_allowed_with_ignore_extra_setting(clust node=node, table=mt, part=part_2020, dest=iceberg, extra_settings=( f"export_merge_tree_part_schema_match_mode = '{schema_match_mode}', " - f"ignore_extra_source_columns = 1" + f"export_merge_tree_part_ignore_extra_source_columns = 1" ), ) wait_for_export_part(node=node, table=mt, part=part_2020) @@ -823,7 +823,7 @@ def test_export_part_column_count_mismatch_source_fewer_still_rejected_with_igno f"SETTINGS allow_experimental_export_merge_tree_part = 1, " f"allow_experimental_insert_into_iceberg = 1, " f"export_merge_tree_part_schema_match_mode = '{schema_match_mode}', " - f"ignore_extra_source_columns = 1" + f"export_merge_tree_part_ignore_extra_source_columns = 1" ) assert expected_error in error, f"Expected {expected_error} with {schema_match_mode}, got: {error!r}" @@ -877,7 +877,7 @@ def test_export_part_ignore_extra_column_breaks_hybrid_over_source_and_destinati export_part( node=node, table=mt, part=part, dest=iceberg, - extra_settings="export_merge_tree_part_schema_match_mode = 'match_by_position', ignore_extra_source_columns = 1", + extra_settings="export_merge_tree_part_schema_match_mode = 'match_by_position', export_merge_tree_part_ignore_extra_source_columns = 1", ) wait_for_export_part(node=node, table=mt, part=part) @@ -972,7 +972,7 @@ def test_export_part_ignore_extra_setting_drops_trailing_alias_column(cluster, s node=node, table=mt, part=part_2020, dest=iceberg, extra_settings=( f"export_merge_tree_part_schema_match_mode = '{schema_match_mode}', " - f"ignore_extra_source_columns = 1" + f"export_merge_tree_part_ignore_extra_source_columns = 1" ), ) wait_for_export_part(node=node, table=mt, part=part_2020) @@ -1007,7 +1007,7 @@ def test_export_part_ignore_extra_setting_kept_alias_depends_on_dropped_column(c node=node, table=mt, part=part_2020, dest=iceberg, extra_settings=( f"export_merge_tree_part_schema_match_mode = '{schema_match_mode}', " - f"ignore_extra_source_columns = 1" + f"export_merge_tree_part_ignore_extra_source_columns = 1" ), ) wait_for_export_part(node=node, table=mt, part=part_2020) @@ -1083,7 +1083,7 @@ def test_export_part_reordered_subset_requires_matching_by_name(cluster): f"SETTINGS allow_experimental_export_merge_tree_part = 1, " f"allow_experimental_insert_into_iceberg = 1, " f"export_merge_tree_part_schema_match_mode = 'match_by_position', " - f"ignore_extra_source_columns = 1" + f"export_merge_tree_part_ignore_extra_source_columns = 1" ) assert "INCOMPATIBLE_COLUMNS" in error, f"Expected positional matching to fail, got: {error!r}" assert node.query(f"SELECT count() FROM {iceberg}").strip() == "0" @@ -1093,7 +1093,7 @@ def test_export_part_reordered_subset_requires_matching_by_name(cluster): mt, part_2020, iceberg, - extra_settings="export_merge_tree_part_schema_match_mode = 'match_by_name', ignore_extra_source_columns = 1", + extra_settings="export_merge_tree_part_schema_match_mode = 'match_by_name', export_merge_tree_part_ignore_extra_source_columns = 1", ) wait_for_export_part(node, mt, part_2020) @@ -1129,7 +1129,7 @@ def test_export_part_match_by_name_requires_every_destination_column(cluster): f"SETTINGS allow_experimental_export_merge_tree_part = 1, " f"allow_experimental_insert_into_iceberg = 1, " f"export_merge_tree_part_schema_match_mode = 'match_by_name', " - f"ignore_extra_source_columns = 1" + f"export_merge_tree_part_ignore_extra_source_columns = 1" ) assert "THERE_IS_NO_COLUMN" in error and "renamed_id" in error, ( f"Expected name matching to reject missing column `renamed_id`, got: {error!r}" diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index 72e0ccc11217..a571625f2827 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -1068,7 +1068,7 @@ def test_export_part_column_count_mismatch_source_fewer_still_rejected_with_igno error = node.query_and_get_error( f"ALTER TABLE {mt_table} EXPORT PART '2020_1_1_0' TO TABLE {s3_table} " f"SETTINGS export_merge_tree_part_schema_match_mode = '{schema_match_mode}', " - f"ignore_extra_source_columns = 1" + f"export_merge_tree_part_ignore_extra_source_columns = 1" ) assert expected_error in error, ( f"Expected {expected_error} for source Date: Mon, 24 Aug 2026 14:00:29 +0200 Subject: [PATCH 09/11] add check before building DAG Signed-off-by: Konstantin Morozov --- src/Common/FailPoint.cpp | 1 + src/Storages/MergeTree/ExportPartTask.cpp | 9 +++ .../MergeTree/ExportPartitionUtils.cpp | 41 +++++++---- src/Storages/MergeTree/ExportPartitionUtils.h | 5 ++ .../test.py | 73 +++++++++++++++++++ 5 files changed, 114 insertions(+), 15 deletions(-) diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index b8fee32bdf13..d7ccfac38631 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -144,6 +144,7 @@ static struct InitFiu REGULAR(export_partition_processed_paths_sync_fail) \ REGULAR(export_part_non_retryable_throw) \ REGULAR(export_part_retryable_throw) \ + PAUSEABLE_ONCE(export_part_pause_before_schema_validation) \ ONCE(backup_add_empty_memory_table) \ PAUSEABLE_ONCE(backup_pause_on_start) \ PAUSEABLE_ONCE(restore_pause_on_start) \ diff --git a/src/Storages/MergeTree/ExportPartTask.cpp b/src/Storages/MergeTree/ExportPartTask.cpp index 7cb2f051f2a5..3df30df41bbd 100644 --- a/src/Storages/MergeTree/ExportPartTask.cpp +++ b/src/Storages/MergeTree/ExportPartTask.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -50,6 +51,7 @@ namespace ErrorCodes namespace FailPoints { + extern const char export_part_pause_before_schema_validation[]; /// Throw a non-retryable (denylisted) error from the part-export worker, so the whole /// export task transitions to FAILED immediately regardless of any timeout. extern const char export_part_non_retryable_throw[]; @@ -119,6 +121,8 @@ namespace const IStorage & destination_storage, const ContextPtr & local_context) { + FailPointInjection::pauseFailPoint(FailPoints::export_part_pause_before_schema_validation); + const auto destination_header = destination_storage.getInMemoryMetadataPtr()->getSampleBlockNonMaterialized(); const auto & destination_columns = destination_header.getColumnsWithTypeAndName(); @@ -132,6 +136,11 @@ namespace auto source_columns = plan_for_part.getCurrentHeader()->getColumnsWithTypeAndName(); const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); + ExportPartitionUtils::checkExportSchemaColumnsCount( + source_columns.size(), + destination_columns.size(), + ignore_extra_source_columns); + if (!match_by_name && ignore_extra_source_columns && src_has_extra_columns) { LOG_DEBUG(getLogger("ExportPartTask"), diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 544a7b9666da..b1e7ef4d86da 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -809,6 +809,28 @@ namespace ExportPartitionUtils UNREACHABLE(); } + void checkExportSchemaColumnsCount( + size_t source_columns_count, + size_t destination_columns_count, + bool ignore_extra_source_columns) + { + if (source_columns_count < destination_columns_count) + throw Exception( + ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, + "Number of columns doesn't match (source: {} and result: {}): " + "destination cannot have more columns than source", + source_columns_count, + destination_columns_count); + + if (source_columns_count > destination_columns_count && !ignore_extra_source_columns) + throw Exception( + ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, + "Number of columns doesn't match (source: {} and result: {}): " + "source has extra columns and the `export_merge_tree_part_ignore_extra_source_columns` setting is disabled", + source_columns_count, + destination_columns_count); + } + void verifyExportSchemaCastable( const StorageMetadataPtr & source_metadata, const StorageMetadataPtr & destination_metadata, @@ -833,21 +855,10 @@ namespace ExportPartitionUtils const bool match_by_name = schema_match_mode == MergeTreePartExportSchemaMatchMode::match_by_name; const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); - if (source_columns.size() < destination_columns.size()) - throw Exception( - ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, - "Number of columns doesn't match (source: {} and result: {}): " - "destination cannot have more columns than source", - source_columns.size(), - destination_columns.size()); - - if (src_has_extra_columns && !ignore_extra_source_columns) - throw Exception( - ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, - "Number of columns doesn't match (source: {} and result: {}): " - "source has extra columns and the `export_merge_tree_part_ignore_extra_source_columns` setting is disabled", - source_columns.size(), - destination_columns.size()); + checkExportSchemaColumnsCount( + source_columns.size(), + destination_columns.size(), + ignore_extra_source_columns); if (!match_by_name && ignore_extra_source_columns && src_has_extra_columns) { diff --git a/src/Storages/MergeTree/ExportPartitionUtils.h b/src/Storages/MergeTree/ExportPartitionUtils.h index d0ba7563ac0d..10a59186867c 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.h +++ b/src/Storages/MergeTree/ExportPartitionUtils.h @@ -101,6 +101,11 @@ namespace ExportPartitionUtils MergeTreePartExportSchemaMatchMode schema_match_mode, const StorageID & destination_storage_id); + void checkExportSchemaColumnsCount( + size_t source_columns_count, + size_t destination_columns_count, + bool ignore_extra_source_columns); + /// Validates that source columns can be exported into the destination with the configured /// positional or name-based CAST matching (`export_merge_tree_part_schema_match_mode`) and /// unmatched-column policy (`export_merge_tree_part_ignore_extra_source_columns`). Lossy casts are rejected unless diff --git a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py index 05b6cfbd1d95..e9753e8825d6 100644 --- a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py +++ b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py @@ -1308,6 +1308,79 @@ def test_export_part_runtime_cast_failure_propagates_async(cluster): node.query(f"DROP TABLE IF EXISTS {iceberg}") +def test_export_part_match_by_name_revalidates_extra_source_columns_in_background_task(cluster): + node = cluster.instances["node1"] + sfx = unique_suffix() + mt = f"mt_match_by_name_schema_drift_{sfx}" + iceberg = f"iceberg_match_by_name_schema_drift_{sfx}" + + make_mt(node, mt, "id Int32, year Int32, payload String", "year") + make_iceberg_s3(node, iceberg, "id Int32, year Int32, payload String", "year") + node.query(f"INSERT INTO {mt} VALUES (1, 2020, 'foo'), (2, 2020, 'bar')") + part = get_part(node, mt, "2020") + + try: + node.query("SYSTEM ENABLE FAILPOINT export_part_pause_before_schema_validation") + export_part( + node, + mt, + part, + iceberg, + "export_merge_tree_part_schema_match_mode = 'match_by_name'", + ) + node.query("SYSTEM WAIT FAILPOINT export_part_pause_before_schema_validation PAUSE") + + node.query( + f"ALTER TABLE {iceberg} DROP COLUMN payload", + settings={"allow_insert_into_iceberg": 1}, + ) + node.query("SYSTEM NOTIFY FAILPOINT export_part_pause_before_schema_validation") + + exception = wait_for_failed_export_part(node, mt, part) + assert "NUMBER_OF_COLUMNS_DOESNT_MATCH" in exception, ( + f"Expected the background schema check to reject the extra source column, got: {exception}" + ) + assert node.query(f"SELECT count() FROM {iceberg}").strip() == "0" + finally: + node.query("SYSTEM DISABLE FAILPOINT export_part_pause_before_schema_validation") + node.query(f"DROP TABLE IF EXISTS {mt} SYNC") + node.query(f"DROP TABLE IF EXISTS {iceberg}") + + +def test_export_part_match_by_name_uses_source_snapshot_when_source_column_is_added(cluster): + node = cluster.instances["node1"] + sfx = unique_suffix() + mt = f"mt_match_by_name_source_schema_drift_{sfx}" + iceberg = f"iceberg_match_by_name_source_schema_drift_{sfx}" + + make_mt(node, mt, "id Int32, year Int32, payload String", "year") + make_iceberg_s3(node, iceberg, "id Int32, year Int32, payload String", "year") + node.query(f"INSERT INTO {mt} VALUES (1, 2020, 'foo'), (2, 2020, 'bar')") + part = get_part(node, mt, "2020") + + try: + node.query("SYSTEM ENABLE FAILPOINT export_part_pause_before_schema_validation") + export_part( + node, + mt, + part, + iceberg, + "export_merge_tree_part_schema_match_mode = 'match_by_name'", + ) + node.query("SYSTEM WAIT FAILPOINT export_part_pause_before_schema_validation PAUSE") + + node.query(f"ALTER TABLE {mt} ADD COLUMN extra String DEFAULT 'new'") + node.query("SYSTEM NOTIFY FAILPOINT export_part_pause_before_schema_validation") + + wait_for_export_part(node, mt, part) + result = node.query(f"SELECT id, year, payload FROM {iceberg} ORDER BY id").strip() + assert result == "1\t2020\tfoo\n2\t2020\tbar", f"Unexpected exported data:\n{result}" + finally: + node.query("SYSTEM DISABLE FAILPOINT export_part_pause_before_schema_validation") + node.query(f"DROP TABLE IF EXISTS {mt} SYNC") + node.query(f"DROP TABLE IF EXISTS {iceberg}") + + def test_export_part_tuple_subcolumn_partition_key_iceberg_rejected(cluster): node = cluster.instances["node1"] sfx = unique_suffix() From 60c1f4f1306aefef0d229683a65458061a3f438d Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Mon, 24 Aug 2026 16:52:50 +0200 Subject: [PATCH 10/11] apply comments: update doc and names Signed-off-by: Konstantin Morozov --- docs/en/antalya/part_export.md | 23 ++++++---------- docs/en/antalya/partition_export.md | 23 ++++++---------- src/Core/Settings.cpp | 10 +++---- src/Core/SettingsChangesHistory.cpp | 2 +- src/Core/SettingsEnums.h | 4 +-- ...portReplicatedMergeTreePartitionManifest.h | 2 +- src/Storages/MergeTree/ExportPartTask.cpp | 2 +- .../MergeTree/ExportPartitionUtils.cpp | 14 +++++----- .../tests/gtest_export_partition_ordering.cpp | 20 +++++++------- .../test.py | 26 +++++++++---------- .../test.py | 12 ++++----- .../test.py | 26 +++++++++---------- .../test.py | 8 +++--- 13 files changed, 79 insertions(+), 93 deletions(-) diff --git a/docs/en/antalya/part_export.md b/docs/en/antalya/part_export.md index fd62919a247b..7bbe4517743f 100644 --- a/docs/en/antalya/part_export.md +++ b/docs/en/antalya/part_export.md @@ -49,13 +49,13 @@ SETTINGS allow_experimental_export_merge_tree_part = 1 Source and destination tables must support positional schema conversion. The following differences between the two schemas are allowed: -- **Column names** may differ between source and destination for non-partition-key columns when `export_merge_tree_part_schema_match_mode = 'match_by_position'` (the default) - columns are matched by position, similar to `INSERT INTO dest SELECT * FROM src`, not by name. Set `export_merge_tree_part_schema_match_mode = 'match_by_name'` to match columns by their exact, case-sensitive name instead, allowing destination columns to be declared in a different order than the source. +- **Column names** may differ between source and destination for non-partition-key columns when `export_merge_tree_part_schema_match_mode = 'POSITION'` (the default) - columns are matched by position, similar to `INSERT INTO dest SELECT * FROM src`, not by name. Set `export_merge_tree_part_schema_match_mode = 'NAME'` to match columns by their exact, case-sensitive name instead, allowing destination columns to be declared in a different order than the source. - **Column types** may differ, as long as the source type is safely castable to the destination type. Set `export_merge_tree_part_allow_lossy_cast = 1` to also permit lossy casts. - **`Tuple` element names** may differ if either the source or destination declares the tuple without named elements: an unnamed `Tuple` (e.g. `Tuple(Int32, Int32)`) is matched against the destination by element position and type only, not by name. For example, exporting from `t Tuple(Int32, Int32)` to `t Tuple(x Int32, y Int32)` is allowed as long as element types match positionally. The following must match between source and destination: -1. **Column count** - by default (`export_merge_tree_part_ignore_extra_source_columns = false`) every source column must have a corresponding destination column: with `export_merge_tree_part_schema_match_mode = 'match_by_position'` (the default) the source and destination must have the same number of columns; with `'match_by_name'` they must have the same set of column names. A mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. Set `export_merge_tree_part_ignore_extra_source_columns = 1` to allow a source table with columns that have no corresponding destination column; such columns are dropped and not exported. The destination having a column absent from the source is always rejected, regardless of this setting. +1. **Column count** - by default, every source column must have a corresponding destination column, and vice versa; a mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. Which source column corresponds to which destination column is determined by `export_merge_tree_part_schema_match_mode`. Set `export_merge_tree_part_ignore_extra_source_columns = 1` to relax this in one direction: a source column without a corresponding destination column is dropped and not exported, instead of throwing. The destination having a column absent from the source is always rejected, regardless of this setting. 2. **`PARTITION BY` expressions** - for destinations other than data lakes, the source and destination `PARTITION BY` expressions must be identical. For Apache Iceberg destinations, the source partition key must be representable as an Iceberg partition spec and must match the destination partition fields and transforms. 3. **The position of every column backing the partition key** - it is not enough for the `PARTITION BY` expressions to be textually identical: every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. If such a column contains a named `Tuple`, its element names must also be declared in the same order (an unnamed `Tuple` on either side is exempt from this, per the allowance above). This comparison is recursive through nested tuples and through container types such as `Array` and `Map`. @@ -139,10 +139,10 @@ In case a table function is used as the destination, the schema can be omitted a ### `export_merge_tree_part_schema_match_mode` (Optional) - **Type**: `MergeTreePartExportSchemaMatchMode` -- **Default**: `match_by_position` +- **Default**: `POSITION` - **Description**: Controls how `EXPORT PART`/`EXPORT PARTITION` matches source `MergeTree` columns to destination columns. Possible values: - - `match_by_position` (default) - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. Column names are not otherwise considered. - - `match_by_name` - every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be declared in a different order than the source. A destination column absent from the source, including when it was renamed, throws `THERE_IS_NO_COLUMN`; there is no positional fallback. + - `POSITION` (default) - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. Column names are not otherwise considered. + - `NAME` - every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be declared in a different order than the source. A destination column absent from the source, including when it was renamed, throws `THERE_IS_NO_COLUMN`; there is no positional fallback. See `export_merge_tree_part_ignore_extra_source_columns` below for how a source column without a corresponding destination column is handled in each mode. @@ -150,19 +150,12 @@ In case a table function is used as the destination, the schema can be omitted a - **Type**: `Bool` - **Default**: `false` -- **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` tolerates source `MergeTree` columns that have no corresponding destination column. - - `false` (default) - such a source column is rejected: the source and destination must match exactly. With `export_merge_tree_part_schema_match_mode = 'match_by_position'` this means the same number of columns; with `'match_by_name'` this means the same set of column names, so a source table with columns absent from the destination is rejected even if the matched columns would otherwise be compatible. A mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - - `true` - a source column without a corresponding destination column is dropped and not exported, instead of throwing. In `match_by_position` mode, this allows a source with extra trailing columns (the destination having more columns than the source is still always rejected). In `match_by_name` mode, this allows source columns whose name has no destination counterpart, which may occur in any position. +- **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` tolerates a source `MergeTree` column that has no corresponding destination column. + - `false` (default) - such a source column is rejected: the source and destination must match exactly. A mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. + - `true` - a source column without a corresponding destination column is dropped and not exported, instead of throwing. The destination having a column absent from the source is still always rejected. Extra source columns are still read and evaluated (including `MATERIALIZED`/`ALIAS` columns, and any column another kept column's `ALIAS`/`MATERIALIZED` expression depends on) before being dropped, so this setting only changes which columns end up in the destination, not what is computed while reading the part. Type conversion and `export_merge_tree_part_allow_lossy_cast` are applied after columns are matched. - Error behavior: - - - With `export_merge_tree_part_ignore_extra_source_columns = false` (default), a source column without a corresponding destination column throws `NUMBER_OF_COLUMNS_DOESNT_MATCH` - in `match_by_position` mode this means any column-count mismatch, in `match_by_name` mode this means the source and destination column-name sets differ. - - In `match_by_position` mode, the destination having more columns than the source always throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`, regardless of `export_merge_tree_part_ignore_extra_source_columns`. - - In `match_by_name` mode, a destination column absent from the source (including a renamed one) always throws `THERE_IS_NO_COLUMN`, regardless of `export_merge_tree_part_ignore_extra_source_columns`; there is no positional fallback. - - After columns have been matched successfully, a cast rejected by the export type-safety check throws `INCOMPATIBLE_COLUMNS`. - ## Examples diff --git a/docs/en/antalya/partition_export.md b/docs/en/antalya/partition_export.md index ff8c47c7a110..acfbf57261c6 100644 --- a/docs/en/antalya/partition_export.md +++ b/docs/en/antalya/partition_export.md @@ -45,9 +45,9 @@ TO TABLE [destination_database.]destination_table ## Requirements -`EXPORT PARTITION` exports each part via the same mechanism as [`EXPORT PART`](/docs/en/antalya/part_export.md#requirements), so the source and destination tables must satisfy the same compatibility requirements. Columns are matched by position by default, or by their exact, case-sensitive name if `export_merge_tree_part_schema_match_mode = 'match_by_name'` is set, and column types may differ as long as they are safely castable (or `export_merge_tree_part_allow_lossy_cast = 1` is set). Beyond that, the following must match: +`EXPORT PARTITION` exports each part via the same mechanism as [`EXPORT PART`](/docs/en/antalya/part_export.md#requirements), so the source and destination tables must satisfy the same compatibility requirements. Columns are matched by position by default, or by their exact, case-sensitive name if `export_merge_tree_part_schema_match_mode = 'NAME'` is set, and column types may differ as long as they are safely castable (or `export_merge_tree_part_allow_lossy_cast = 1` is set). Beyond that, the following must match: -1. **Column count** - by default (`export_merge_tree_part_ignore_extra_source_columns = false`) every source column must have a corresponding destination column: with `export_merge_tree_part_schema_match_mode = 'match_by_position'` (the default) the source and destination must have the same number of columns; with `'match_by_name'` they must have the same set of column names. Set `export_merge_tree_part_ignore_extra_source_columns = 1` to allow a source table with columns that have no corresponding destination column; the destination having a column absent from the source is still always rejected. +1. **Column count** - by default, every source column must have a corresponding destination column, and vice versa; a mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. Which source column corresponds to which destination column is determined by `export_merge_tree_part_schema_match_mode`. Set `export_merge_tree_part_ignore_extra_source_columns = 1` to relax this in one direction: a source column without a corresponding destination column is dropped and not exported, instead of throwing. The destination having a column absent from the source is always rejected, regardless of this setting. 2. **`PARTITION BY` expressions** - for destinations other than data lakes, the source and destination `PARTITION BY` expressions must be identical. For Apache Iceberg destinations, the source partition key must match the destination partition fields and transforms. 3. **Partition key column positions and layouts** - every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. Named `Tuple` elements within such a column must also be declared in the same order, including tuples nested inside `Array` or `Map`. This applies even if both tables' `PARTITION BY` expressions are textually identical. See [`EXPORT PART` requirements](/docs/en/antalya/part_export.md#requirements) for a worked example and the corresponding exception message. @@ -131,10 +131,10 @@ Notes: ### `export_merge_tree_part_schema_match_mode` (Optional) - **Type**: `MergeTreePartExportSchemaMatchMode` -- **Default**: `match_by_position` +- **Default**: `POSITION` - **Description**: Controls how `EXPORT PART`/`EXPORT PARTITION` matches source `MergeTree` columns to destination columns. Possible values: - - `match_by_position` (default) - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. Column names are not otherwise considered. - - `match_by_name` - every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be declared in a different order than the source. A destination column absent from the source, including when it was renamed, throws `THERE_IS_NO_COLUMN`; there is no positional fallback. + - `POSITION` (default) - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. Column names are not otherwise considered. + - `NAME` - every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be declared in a different order than the source. A destination column absent from the source, including when it was renamed, throws `THERE_IS_NO_COLUMN`; there is no positional fallback. See `export_merge_tree_part_ignore_extra_source_columns` below for how a source column without a corresponding destination column is handled in each mode. @@ -142,19 +142,12 @@ Notes: - **Type**: `Bool` - **Default**: `false` -- **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` tolerates source `MergeTree` columns that have no corresponding destination column. - - `false` (default) - such a source column is rejected: the source and destination must match exactly. With `export_merge_tree_part_schema_match_mode = 'match_by_position'` this means the same number of columns; with `'match_by_name'` this means the same set of column names, so a source table with columns absent from the destination is rejected even if the matched columns would otherwise be compatible. A mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - - `true` - a source column without a corresponding destination column is dropped and not exported, instead of throwing. In `match_by_position` mode, this allows a source with extra trailing columns (the destination having more columns than the source is still always rejected). In `match_by_name` mode, this allows source columns whose name has no destination counterpart, which may occur in any position. +- **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` tolerates a source `MergeTree` column that has no corresponding destination column. + - `false` (default) - such a source column is rejected: the source and destination must match exactly. A mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. + - `true` - a source column without a corresponding destination column is dropped and not exported, instead of throwing. The destination having a column absent from the source is still always rejected. Extra source columns are still read and evaluated (including `MATERIALIZED`/`ALIAS` columns, and any column another kept column's `ALIAS`/`MATERIALIZED` expression depends on) before being dropped, so this setting only changes which columns end up in the destination, not what is computed while reading the part. Type conversion and `export_merge_tree_part_allow_lossy_cast` are applied after columns are matched. - Error behavior: - - - With `export_merge_tree_part_ignore_extra_source_columns = false` (default), a source column without a corresponding destination column throws `NUMBER_OF_COLUMNS_DOESNT_MATCH` - in `match_by_position` mode this means any column-count mismatch, in `match_by_name` mode this means the source and destination column-name sets differ. - - In `match_by_position` mode, the destination having more columns than the source always throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`, regardless of `export_merge_tree_part_ignore_extra_source_columns`. - - In `match_by_name` mode, a destination column absent from the source (including a renamed one) always throws `THERE_IS_NO_COLUMN`, regardless of `export_merge_tree_part_ignore_extra_source_columns`; there is no positional fallback. - - After columns have been matched successfully, a cast rejected by the export type-safety check throws `INCOMPATIBLE_COLUMNS`. - ## Examples ### Basic Export to S3 diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index ec9efc301dad..47e99d476acd 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -7627,18 +7627,18 @@ Allow `EXPORT PART`/`EXPORT PARTITION` to apply lossy (non-value-preserving) cas When exporting to Apache Iceberg, the partition value written to the metadata is derived from the source partition columns by casting them to the destination partition-field types and applying the destination partition transform — the same computation the exported data files use, so the metadata stays consistent with the data. A lossy cast on a partition column remains semantically truncating: both the data files and the metadata contain the truncated value, and such casts require this setting to be enabled. )", 0) \ - DECLARE(MergeTreePartExportSchemaMatchMode, export_merge_tree_part_schema_match_mode, MergeTreePartExportSchemaMatchMode::match_by_position, R"( + DECLARE(MergeTreePartExportSchemaMatchMode, export_merge_tree_part_schema_match_mode, MergeTreePartExportSchemaMatchMode::POSITION, R"( Controls how `EXPORT PART`/`EXPORT PARTITION` matches source `MergeTree` columns to destination columns. Possible values: -- `match_by_position` (default) - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. Column names are not otherwise considered. -- `match_by_name` - every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be declared in a different order than the source. A destination column absent from the source throws `THERE_IS_NO_COLUMN`, with no positional fallback. +- `POSITION` (default) - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. Column names are not otherwise considered. +- `NAME` - every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be declared in a different order than the source. A destination column absent from the source throws `THERE_IS_NO_COLUMN`, with no positional fallback. See also `export_merge_tree_part_ignore_extra_source_columns`, which controls whether a source column without a corresponding destination column is dropped or rejected. )", 0) \ DECLARE(Bool, export_merge_tree_part_ignore_extra_source_columns, false, R"( Controls whether `EXPORT PART`/`EXPORT PARTITION` tolerates source `MergeTree` columns that have no corresponding destination column. -- `false` (default) - such a source column is rejected: the source and destination must match exactly (in `export_merge_tree_part_schema_match_mode = 'match_by_position'`, this means the same number of columns; in `'match_by_name'`, the same set of column names). A mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. -- `true` - a source column without a corresponding destination column is dropped and not exported, instead of throwing. In `match_by_position` mode, this allows a source with extra trailing columns (the destination having more columns than the source is still always rejected). In `match_by_name` mode, this allows source columns whose name has no destination counterpart. +- `false` (default) - such a source column is rejected: the source and destination must match exactly (in `export_merge_tree_part_schema_match_mode = 'POSITION'`, this means the same number of columns; in `'NAME'`, the same set of column names). A mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. +- `true` - a source column without a corresponding destination column is dropped and not exported, instead of throwing. In `POSITION` mode, this allows a source with extra trailing columns (the destination having more columns than the source is still always rejected). In `NAME` mode, this allows source columns whose name has no destination counterpart. Extra source columns are still read and evaluated (including `MATERIALIZED`/`ALIAS` columns, and any column another kept column's `ALIAS`/`MATERIALIZED` expression depends on) before being dropped, so this setting only changes which columns end up in the destination, not what is computed while reading the part. )", 0) \ diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 26df2661a4a7..3fe09fc73344 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -44,7 +44,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"object_storage_cluster_join_mode", "allow", "allow", "New setting"}, {"export_merge_tree_partition_task_timeout_seconds", "3600", "86400", "Increase default value to make it more realistic"}, {"export_merge_tree_part_allow_lossy_cast", false, false, "New setting to gate lossy casts in EXPORT PART/PARTITION behind explicit acknowledgment"}, - {"export_merge_tree_part_schema_match_mode", "match_by_position", "match_by_position", "New setting to control how EXPORT PART/EXPORT PARTITION matches source columns to destination columns"}, + {"export_merge_tree_part_schema_match_mode", "POSITION", "POSITION", "New setting to control how EXPORT PART/EXPORT PARTITION matches source columns to destination columns"}, {"export_merge_tree_part_ignore_extra_source_columns", false, false, "New setting to allow EXPORT PART/EXPORT PARTITION when the source table has columns absent from the destination"}, {"export_merge_tree_partition_retry_initial_backoff_seconds", 5, 5, "New setting for exponential back-off between failed part export retries in an export partition task"}, {"export_merge_tree_partition_retry_max_backoff_seconds", 300, 300, "New setting capping the exponential back-off between failed part export retries in an export partition task"}, diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index a325835452d5..5544906237ba 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -576,8 +576,8 @@ DECLARE_SETTING_ENUM(MergeTreePartExportFileAlreadyExistsPolicy) enum class MergeTreePartExportSchemaMatchMode : uint8_t { - match_by_position, - match_by_name, + POSITION, + NAME, }; DECLARE_SETTING_ENUM(MergeTreePartExportSchemaMatchMode) diff --git a/src/Storages/ExportReplicatedMergeTreePartitionManifest.h b/src/Storages/ExportReplicatedMergeTreePartitionManifest.h index 791691b849da..b973d31398ba 100644 --- a/src/Storages/ExportReplicatedMergeTreePartitionManifest.h +++ b/src/Storages/ExportReplicatedMergeTreePartitionManifest.h @@ -366,7 +366,7 @@ struct ExportReplicatedMergeTreePartitionManifest /// Left unset (nullopt) for tasks created before these fields existed - such tasks were /// always scheduled under the old, strict column-matching check (a mismatch could never /// reach scheduling in the first place), so callers should treat an absent value as - /// `match_by_position` with `ignore_extra_source_columns = false`. + /// `POSITION` with `ignore_extra_source_columns = false`. if (json->has("schema_match_mode")) { const auto schema_match_mode = magic_enum::enum_cast(json->getValue("schema_match_mode")); diff --git a/src/Storages/MergeTree/ExportPartTask.cpp b/src/Storages/MergeTree/ExportPartTask.cpp index 3df30df41bbd..ca6ab0a9ecc3 100644 --- a/src/Storages/MergeTree/ExportPartTask.cpp +++ b/src/Storages/MergeTree/ExportPartTask.cpp @@ -131,7 +131,7 @@ namespace local_context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value; const bool ignore_extra_source_columns = local_context->getSettingsRef()[Setting::export_merge_tree_part_ignore_extra_source_columns]; - const bool match_by_name = schema_match_mode == MergeTreePartExportSchemaMatchMode::match_by_name; + const bool match_by_name = schema_match_mode == MergeTreePartExportSchemaMatchMode::NAME; auto source_columns = plan_for_part.getCurrentHeader()->getColumnsWithTypeAndName(); const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index b1e7ef4d86da..6915c2884223 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -178,11 +178,11 @@ namespace ExportPartitionUtils context_copy->setSetting("output_format_parquet_row_group_size_bytes", *manifest.parquet_row_group_size_bytes); /// Manifests written before these settings existed have no value here; such tasks were always /// scheduled under the old, strict column-matching check, so an absent value must resolve to - /// `match_by_position` / `false` regardless of the ambient context's settings (which may have + /// `POSITION` / `false` regardless of the ambient context's settings (which may have /// since been changed). context_copy->setSetting( "export_merge_tree_part_schema_match_mode", - String(magic_enum::enum_name(manifest.schema_match_mode.value_or(MergeTreePartExportSchemaMatchMode::match_by_position)))); + String(magic_enum::enum_name(manifest.schema_match_mode.value_or(MergeTreePartExportSchemaMatchMode::POSITION)))); context_copy->setSetting( "export_merge_tree_part_ignore_extra_source_columns", manifest.ignore_extra_source_columns.value_or(false)); @@ -783,12 +783,12 @@ namespace ExportPartitionUtils { switch (schema_match_mode) { - case MergeTreePartExportSchemaMatchMode::match_by_position: { + case MergeTreePartExportSchemaMatchMode::POSITION: { for (size_t i = 0; i < destination_columns.size(); ++i) verifyExportColumnCastIsSafe(source_columns[i], destination_columns[i], destination_storage_id); return; } - case MergeTreePartExportSchemaMatchMode::match_by_name: { + case MergeTreePartExportSchemaMatchMode::NAME: { std::unordered_map source_columns_by_name; source_columns_by_name.reserve(source_columns.size()); for (const auto & source_column : source_columns) @@ -852,7 +852,7 @@ namespace ExportPartitionUtils context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value; const bool ignore_extra_source_columns = context->getSettingsRef()[Setting::export_merge_tree_part_ignore_extra_source_columns]; - const bool match_by_name = schema_match_mode == MergeTreePartExportSchemaMatchMode::match_by_name; + const bool match_by_name = schema_match_mode == MergeTreePartExportSchemaMatchMode::NAME; const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); checkExportSchemaColumnsCount( @@ -895,7 +895,7 @@ namespace ExportPartitionUtils switch (schema_match_mode) { - case MergeTreePartExportSchemaMatchMode::match_by_name: + case MergeTreePartExportSchemaMatchMode::NAME: { std::unordered_map source_positions_by_name; source_positions_by_name.reserve(source_columns.size()); @@ -917,7 +917,7 @@ namespace ExportPartitionUtils } break; } - case MergeTreePartExportSchemaMatchMode::match_by_position: + case MergeTreePartExportSchemaMatchMode::POSITION: { const size_t num_columns = std::min(source_columns.size(), destination_columns.size()); for (size_t i = 0; i < num_columns; ++i) diff --git a/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp b/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp index b01c564e63d9..c31f4908f28f 100644 --- a/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp +++ b/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp @@ -149,7 +149,7 @@ TEST_F(ExportPartitionOrderingTest, IterationOrderMatchesCreateTime) TEST_F(ExportPartitionManifestBackCompatTest, MissingSchemaMatchModeParsesAsNullopt) { auto manifest = makeValidManifest(); - manifest.schema_match_mode = MergeTreePartExportSchemaMatchMode::match_by_name; + manifest.schema_match_mode = MergeTreePartExportSchemaMatchMode::NAME; Poco::JSON::Parser parser; auto json = parser.parse(manifest.toJsonString()).extract(); @@ -216,7 +216,7 @@ TEST_F(ExportPartitionManifestBackCompatTest, MissingSchemaMatchSettingsFallBack EXPECT_EQ( worker_context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value, - MergeTreePartExportSchemaMatchMode::match_by_position); + MergeTreePartExportSchemaMatchMode::POSITION); EXPECT_EQ( worker_context->getSettingsRef()[Setting::export_merge_tree_part_ignore_extra_source_columns].value, false); @@ -273,7 +273,7 @@ TEST(ExportColumnCastsTest, UsesSelectedMatchingMode) ExportPartitionUtils::verifyExportColumnCastsAreSafe( source_columns, destination_columns, - MergeTreePartExportSchemaMatchMode::match_by_position, + MergeTreePartExportSchemaMatchMode::POSITION, destination_storage_id); }, ErrorCodes::INCOMPATIBLE_COLUMNS); @@ -281,7 +281,7 @@ TEST(ExportColumnCastsTest, UsesSelectedMatchingMode) EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( source_columns, destination_columns, - MergeTreePartExportSchemaMatchMode::match_by_name, + MergeTreePartExportSchemaMatchMode::NAME, destination_storage_id)); } @@ -305,7 +305,7 @@ TEST(ExportColumnCastsTest, MatchByNameAcceptsReorderedColumnsWithEqualColumnCou ExportPartitionUtils::verifyExportColumnCastsAreSafe( source_columns, reordered_destination_columns, - MergeTreePartExportSchemaMatchMode::match_by_position, + MergeTreePartExportSchemaMatchMode::POSITION, destination_storage_id); }, ErrorCodes::INCOMPATIBLE_COLUMNS); @@ -313,7 +313,7 @@ TEST(ExportColumnCastsTest, MatchByNameAcceptsReorderedColumnsWithEqualColumnCou EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( source_columns, reordered_destination_columns, - MergeTreePartExportSchemaMatchMode::match_by_name, + MergeTreePartExportSchemaMatchMode::NAME, destination_storage_id)); const ColumnsWithTypeAndName same_order_destination_columns = { @@ -322,7 +322,7 @@ TEST(ExportColumnCastsTest, MatchByNameAcceptsReorderedColumnsWithEqualColumnCou makeColumn("payload"), }; - for (const auto mode : {MergeTreePartExportSchemaMatchMode::match_by_position, MergeTreePartExportSchemaMatchMode::match_by_name}) + for (const auto mode : {MergeTreePartExportSchemaMatchMode::POSITION, MergeTreePartExportSchemaMatchMode::NAME}) EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( source_columns, same_order_destination_columns, mode, destination_storage_id)); } @@ -343,7 +343,7 @@ TEST(ExportColumnCastsTest, MatchByNameToleratesUnmatchedSourceColumn) EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( source_columns, destination_columns, - MergeTreePartExportSchemaMatchMode::match_by_name, + MergeTreePartExportSchemaMatchMode::NAME, destination_storage_id)); } @@ -365,7 +365,7 @@ TEST(ExportColumnCastsTest, RejectsLossyCastAfterMatchingByName) ExportPartitionUtils::verifyExportColumnCastsAreSafe( source_columns, destination_columns, - MergeTreePartExportSchemaMatchMode::match_by_name, + MergeTreePartExportSchemaMatchMode::NAME, StorageID{"test", "destination"}); }, ErrorCodes::INCOMPATIBLE_COLUMNS); @@ -389,7 +389,7 @@ TEST(ExportColumnCastsTest, RejectsMissingDestinationColumnAfterMatchingByName) ExportPartitionUtils::verifyExportColumnCastsAreSafe( source_columns, destination_columns, - MergeTreePartExportSchemaMatchMode::match_by_name, + MergeTreePartExportSchemaMatchMode::NAME, StorageID{"test", "destination"}); }, ErrorCodes::THERE_IS_NO_COLUMN); diff --git a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py index e9753e8825d6..38169004ddc9 100644 --- a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py +++ b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py @@ -38,8 +38,8 @@ EXTRA_SOURCE_COLUMN_MODES = [ - pytest.param("match_by_position", id="by-position"), - pytest.param("match_by_name", id="by-name"), + pytest.param("POSITION", id="by-position"), + pytest.param("NAME", id="by-name"), ] @@ -800,8 +800,8 @@ def test_export_part_source_more_columns_allowed_with_ignore_extra_setting(clust @pytest.mark.parametrize( "schema_match_mode,expected_error", [ - pytest.param("match_by_position", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-position"), - pytest.param("match_by_name", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-name"), + pytest.param("POSITION", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-position"), + pytest.param("NAME", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-name"), ], ) def test_export_part_column_count_mismatch_source_fewer_still_rejected_with_ignore_extra_setting( @@ -877,7 +877,7 @@ def test_export_part_ignore_extra_column_breaks_hybrid_over_source_and_destinati export_part( node=node, table=mt, part=part, dest=iceberg, - extra_settings="export_merge_tree_part_schema_match_mode = 'match_by_position', export_merge_tree_part_ignore_extra_source_columns = 1", + extra_settings="export_merge_tree_part_schema_match_mode = 'POSITION', export_merge_tree_part_ignore_extra_source_columns = 1", ) wait_for_export_part(node=node, table=mt, part=part) @@ -1082,7 +1082,7 @@ def test_export_part_reordered_subset_requires_matching_by_name(cluster): f"ALTER TABLE {mt} EXPORT PART '{part_2020}' TO TABLE {iceberg} " f"SETTINGS allow_experimental_export_merge_tree_part = 1, " f"allow_experimental_insert_into_iceberg = 1, " - f"export_merge_tree_part_schema_match_mode = 'match_by_position', " + f"export_merge_tree_part_schema_match_mode = 'POSITION', " f"export_merge_tree_part_ignore_extra_source_columns = 1" ) assert "INCOMPATIBLE_COLUMNS" in error, f"Expected positional matching to fail, got: {error!r}" @@ -1093,7 +1093,7 @@ def test_export_part_reordered_subset_requires_matching_by_name(cluster): mt, part_2020, iceberg, - extra_settings="export_merge_tree_part_schema_match_mode = 'match_by_name', export_merge_tree_part_ignore_extra_source_columns = 1", + extra_settings="export_merge_tree_part_schema_match_mode = 'NAME', export_merge_tree_part_ignore_extra_source_columns = 1", ) wait_for_export_part(node, mt, part_2020) @@ -1128,7 +1128,7 @@ def test_export_part_match_by_name_requires_every_destination_column(cluster): f"ALTER TABLE {mt} EXPORT PART '{part_2020}' TO TABLE {iceberg} " f"SETTINGS allow_experimental_export_merge_tree_part = 1, " f"allow_experimental_insert_into_iceberg = 1, " - f"export_merge_tree_part_schema_match_mode = 'match_by_name', " + f"export_merge_tree_part_schema_match_mode = 'NAME', " f"export_merge_tree_part_ignore_extra_source_columns = 1" ) assert "THERE_IS_NO_COLUMN" in error and "renamed_id" in error, ( @@ -1160,7 +1160,7 @@ def test_export_part_match_by_name_rejects_renamed_column_without_extra_source_c f"ALTER TABLE {mt} EXPORT PART '{part_2020}' TO TABLE {iceberg} " f"SETTINGS allow_experimental_export_merge_tree_part = 1, " f"allow_experimental_insert_into_iceberg = 1, " - f"export_merge_tree_part_schema_match_mode = 'match_by_name'" + f"export_merge_tree_part_schema_match_mode = 'NAME'" ) assert "THERE_IS_NO_COLUMN" in error and "renamed_id" in error, ( f"Expected name matching to reject missing column `renamed_id` even without an extra " @@ -1173,7 +1173,7 @@ def test_export_part_match_by_name_rejects_renamed_column_without_extra_source_c def test_export_part_match_by_name_reorders_columns_without_extra_source_columns(cluster): - """The scenario `export_merge_tree_part_schema_match_mode = 'match_by_name'` was introduced for: + """The scenario `export_merge_tree_part_schema_match_mode = 'NAME'` was introduced for: the source and destination have the exact same number of columns, declared in a different order. Previously this fell back to positional matching (a no-op for by-name mode); now it is matched by name like any other case.""" @@ -1190,7 +1190,7 @@ def test_export_part_match_by_name_reorders_columns_without_extra_source_columns export_part( node, mt, part_2020, iceberg, - extra_settings="export_merge_tree_part_schema_match_mode = 'match_by_name'", + extra_settings="export_merge_tree_part_schema_match_mode = 'NAME'", ) wait_for_export_part(node, mt, part_2020) @@ -1326,7 +1326,7 @@ def test_export_part_match_by_name_revalidates_extra_source_columns_in_backgroun mt, part, iceberg, - "export_merge_tree_part_schema_match_mode = 'match_by_name'", + "export_merge_tree_part_schema_match_mode = 'NAME'", ) node.query("SYSTEM WAIT FAILPOINT export_part_pause_before_schema_validation PAUSE") @@ -1365,7 +1365,7 @@ def test_export_part_match_by_name_uses_source_snapshot_when_source_column_is_ad mt, part, iceberg, - "export_merge_tree_part_schema_match_mode = 'match_by_name'", + "export_merge_tree_part_schema_match_mode = 'NAME'", ) node.query("SYSTEM WAIT FAILPOINT export_part_pause_before_schema_validation PAUSE") diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index a571625f2827..44abefeed98d 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -10,8 +10,8 @@ EXTRA_SOURCE_COLUMN_MODES = [ - pytest.param("match_by_position", id="by-position"), - pytest.param("match_by_name", id="by-name"), + pytest.param("POSITION", id="by-position"), + pytest.param("NAME", id="by-name"), ] @@ -1039,8 +1039,8 @@ def test_export_part_column_count_mismatch_source_fewer_is_rejected(cluster): @pytest.mark.parametrize( "schema_match_mode,expected_error", [ - pytest.param("match_by_position", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-position"), - pytest.param("match_by_name", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-name"), + pytest.param("POSITION", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-position"), + pytest.param("NAME", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-name"), ], ) def test_export_part_column_count_mismatch_source_fewer_still_rejected_with_ignore_extra_setting( @@ -1140,7 +1140,7 @@ def test_export_part_match_by_name_with_equal_column_count_reordered(cluster): node.query( f"ALTER TABLE {mt_table} EXPORT PART '2020_1_1_0' TO TABLE {s3_table} " - f"SETTINGS export_merge_tree_part_schema_match_mode = 'match_by_name'" + f"SETTINGS export_merge_tree_part_schema_match_mode = 'NAME'" ) wait_for_export_part(node=node, table=mt_table, part="2020_1_1_0") @@ -1174,7 +1174,7 @@ def test_export_part_match_by_name_with_equal_column_count_rejects_unmatched_sou error = node.query_and_get_error( f"ALTER TABLE {mt_table} EXPORT PART '2020_1_1_0' TO TABLE {s3_table} " - f"SETTINGS export_merge_tree_part_schema_match_mode = 'match_by_name'" + f"SETTINGS export_merge_tree_part_schema_match_mode = 'NAME'" ) assert "NUMBER_OF_COLUMNS_DOESNT_MATCH" in error, ( f"Expected NUMBER_OF_COLUMNS_DOESNT_MATCH for an unmatched source column with " diff --git a/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py b/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py index 911147521bda..d89bf2bd92ba 100644 --- a/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py +++ b/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py @@ -27,8 +27,8 @@ EXTRA_SOURCE_COLUMN_MODES = [ - pytest.param("match_by_position", id="by-position"), - pytest.param("match_by_name", id="by-name"), + pytest.param("POSITION", id="by-position"), + pytest.param("NAME", id="by-name"), ] @@ -1310,8 +1310,8 @@ def test_export_partition_column_count_mismatch_source_fewer_is_rejected(cluster def test_export_partition_source_more_columns_allowed_with_ignore_extra_setting(cluster, schema_match_mode): """ Source has 3 columns (id, year, extra), destination has 2 (id, year). - With `export_merge_tree_part_schema_match_mode` set to either `match_by_position` or - `match_by_name` and `export_merge_tree_part_ignore_extra_source_columns = 1`, the export must succeed: the + With `export_merge_tree_part_schema_match_mode` set to either `POSITION` or + `NAME` and `export_merge_tree_part_ignore_extra_source_columns = 1`, the export must succeed: the trailing `extra` source column is dropped and only `id`/`year` land in the destination. """ node = cluster.instances["replica1"] @@ -1342,11 +1342,11 @@ def test_export_partition_source_more_columns_allowed_with_ignore_extra_setting( f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}", settings={ "allow_insert_into_iceberg": 1, - "export_merge_tree_part_schema_match_mode": "match_by_position", + "export_merge_tree_part_schema_match_mode": "POSITION", }, ) assert "NUMBER_OF_COLUMNS_DOESNT_MATCH" in error, ( - f"Expected NUMBER_OF_COLUMNS_DOESNT_MATCH with schema_match_mode='match_by_position', got: {error!r}" + f"Expected NUMBER_OF_COLUMNS_DOESNT_MATCH with schema_match_mode='POSITION', got: {error!r}" ) node.query( @@ -1370,8 +1370,8 @@ def test_export_partition_source_more_columns_allowed_with_ignore_extra_setting( @pytest.mark.parametrize( "schema_match_mode,expected_error", [ - pytest.param("match_by_position", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-position"), - pytest.param("match_by_name", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-name"), + pytest.param("POSITION", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-position"), + pytest.param("NAME", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-name"), ], ) def test_export_partition_column_count_mismatch_source_fewer_still_rejected_with_ignore_extra_setting( @@ -1379,7 +1379,7 @@ def test_export_partition_column_count_mismatch_source_fewer_still_rejected_with ): """ Setting `export_merge_tree_part_ignore_extra_source_columns = 1` never relaxes the source-has-fewer-columns - direction, in either `match_by_position` or `match_by_name` mode. Source has 2 columns + direction, in either `POSITION` or `NAME` mode. Source has 2 columns (id, year), destination has 3 (id, year, extra): the destination cannot be filled from the source, so this must still be rejected synchronously even with the relaxed setting. """ @@ -1616,7 +1616,7 @@ def test_export_partition_ignore_extra_setting_prefix_contains_different_name(cl f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}", settings={ "allow_insert_into_iceberg": 1, - "export_merge_tree_part_schema_match_mode": "match_by_position", + "export_merge_tree_part_schema_match_mode": "POSITION", "export_merge_tree_part_ignore_extra_source_columns": 1, }, ) @@ -1632,7 +1632,7 @@ def test_export_partition_ignore_extra_setting_prefix_contains_different_name(cl @pytest.mark.parametrize("schema_match_mode", EXTRA_SOURCE_COLUMN_MODES) def test_export_partition_matches_columns_when_column_counts_are_equal(cluster, schema_match_mode): """Source and destination declare the same 2 columns, in the same order and with the same - names, so `match_by_position` and `match_by_name` must agree and both succeed identically - + names, so `POSITION` and `NAME` must agree and both succeed identically - there is no unmatched source column for `export_merge_tree_part_ignore_extra_source_columns` to affect.""" node = cluster.instances["replica1"] @@ -1673,7 +1673,7 @@ def test_export_partition_column_count_mismatch_into_table_with_existing_data(cl ignore_extra_settings = { "allow_insert_into_iceberg": 1, - "export_merge_tree_part_schema_match_mode": "match_by_position", + "export_merge_tree_part_schema_match_mode": "POSITION", "export_merge_tree_part_ignore_extra_source_columns": 1, } @@ -1723,7 +1723,7 @@ def test_export_partition_column_count_mismatch_into_partition_that_already_has_ ignore_extra_settings = { "allow_insert_into_iceberg": 1, - "export_merge_tree_part_schema_match_mode": "match_by_position", + "export_merge_tree_part_schema_match_mode": "POSITION", "export_merge_tree_part_ignore_extra_source_columns": 1, } diff --git a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py index c0d3e8537273..50e853ee1389 100644 --- a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py +++ b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py @@ -16,8 +16,8 @@ EXTRA_SOURCE_COLUMN_MODES = [ - pytest.param("match_by_position", id="by-position"), - pytest.param("match_by_name", id="by-name"), + pytest.param("POSITION", id="by-position"), + pytest.param("NAME", id="by-name"), ] @@ -2029,7 +2029,7 @@ def test_export_partition_match_by_name_honored_by_non_initiating_replica(cluste replica1.query( f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}" - f" SETTINGS export_merge_tree_part_schema_match_mode = 'match_by_name'," + f" SETTINGS export_merge_tree_part_schema_match_mode = 'NAME'," f" export_merge_tree_part_ignore_extra_source_columns = 1" ) @@ -2078,7 +2078,7 @@ def test_export_partition_match_by_name_with_equal_column_count_reordered(cluste replica1.query( f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}" - f" SETTINGS export_merge_tree_part_schema_match_mode = 'match_by_name'" + f" SETTINGS export_merge_tree_part_schema_match_mode = 'NAME'" ) wait_for_export_status(node=replica1, source_table=mt_table, dest_table=s3_table, From f6a0c73726f38c86e972b82bace5bd90f00b022e Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Mon, 24 Aug 2026 17:50:51 +0200 Subject: [PATCH 11/11] add comments Signed-off-by: Konstantin Morozov --- src/Storages/MergeTree/ExportPartTask.cpp | 12 +++++++----- .../MergeTree/ExportPartitionUtils.cpp | 18 +++++++++++------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/Storages/MergeTree/ExportPartTask.cpp b/src/Storages/MergeTree/ExportPartTask.cpp index ca6ab0a9ecc3..82c85f4b6e09 100644 --- a/src/Storages/MergeTree/ExportPartTask.cpp +++ b/src/Storages/MergeTree/ExportPartTask.cpp @@ -131,7 +131,6 @@ namespace local_context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value; const bool ignore_extra_source_columns = local_context->getSettingsRef()[Setting::export_merge_tree_part_ignore_extra_source_columns]; - const bool match_by_name = schema_match_mode == MergeTreePartExportSchemaMatchMode::NAME; auto source_columns = plan_for_part.getCurrentHeader()->getColumnsWithTypeAndName(); const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); @@ -141,7 +140,12 @@ namespace destination_columns.size(), ignore_extra_source_columns); - if (!match_by_name && ignore_extra_source_columns && src_has_extra_columns) + const ActionsDAG::MatchColumnsMode mode = schema_match_mode == MergeTreePartExportSchemaMatchMode::NAME + ? ActionsDAG::MatchColumnsMode::Name + : ActionsDAG::MatchColumnsMode::Position; + + // makeConvertingActions with postitional mode requires equal columns count, + if (ActionsDAG::MatchColumnsMode::Position == mode && ignore_extra_source_columns && src_has_extra_columns) { LOG_DEBUG(getLogger("ExportPartTask"), "Source has {} columns while destination has {} columns, " @@ -173,9 +177,7 @@ namespace auto dag = ActionsDAG::makeConvertingActions( source_columns, destination_columns, - match_by_name - ? ActionsDAG::MatchColumnsMode::Name - : ActionsDAG::MatchColumnsMode::Position, + mode, local_context); auto expression_step = std::make_unique( diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 6915c2884223..f0606799699b 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -852,7 +852,6 @@ namespace ExportPartitionUtils context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value; const bool ignore_extra_source_columns = context->getSettingsRef()[Setting::export_merge_tree_part_ignore_extra_source_columns]; - const bool match_by_name = schema_match_mode == MergeTreePartExportSchemaMatchMode::NAME; const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); checkExportSchemaColumnsCount( @@ -860,12 +859,19 @@ namespace ExportPartitionUtils destination_columns.size(), ignore_extra_source_columns); - if (!match_by_name && ignore_extra_source_columns && src_has_extra_columns) + const ActionsDAG::MatchColumnsMode mode = schema_match_mode == MergeTreePartExportSchemaMatchMode::NAME + ? ActionsDAG::MatchColumnsMode::Name + : ActionsDAG::MatchColumnsMode::Position; + + // makeConvertingActions with postitional mode requires equal columns count, + if (ActionsDAG::MatchColumnsMode::Position == mode && ignore_extra_source_columns && src_has_extra_columns) { - LOG_DEBUG(getLogger("ExportPartitionUtils"), + LOG_DEBUG( + getLogger("ExportPartitionUtils"), "Source has {} columns while destination has {} columns, " "the {} extra trailing source column(s) will be ignored", - source_columns.size(), destination_columns.size(), + source_columns.size(), + destination_columns.size(), source_columns.size() - destination_columns.size()); source_columns.resize(destination_columns.size()); @@ -874,9 +880,7 @@ namespace ExportPartitionUtils (void) ActionsDAG::makeConvertingActions( source_columns, destination_columns, - match_by_name - ? ActionsDAG::MatchColumnsMode::Name - : ActionsDAG::MatchColumnsMode::Position, + mode, context); const auto & source_columns_description = source_metadata->getColumns();