diff --git a/include/paimon/defs.h b/include/paimon/defs.h index 0ca67e3a7..e05faefd8 100644 --- a/include/paimon/defs.h +++ b/include/paimon/defs.h @@ -386,14 +386,34 @@ struct PAIMON_EXPORT Options { /// @note: bitmap64 dv is not supported. static const char DELETION_VECTOR_BITMAP64[]; - /// @note `CHANGELOG_PRODUCER` currently only support `none` - /// /// "changelog-producer" - Whether to double write to a changelog file. This changelog file /// keeps the details of data changes, it can be read directly during stream reads. This can be /// applied to tables with primary keys. Values can be "none", "input", "lookup", /// "full-compaction". Default value is "none". + /// @note C++ Paimon currently supports "none", "input", and "lookup". static const char CHANGELOG_PRODUCER[]; + /// "changelog-producer.row-deduplicate" - Whether to generate update-before and update-after + /// changelog records when the row has not changed. This option is only valid for "lookup" or + /// "full-compaction" changelog producers. Default value is "false". + static const char CHANGELOG_PRODUCER_ROW_DEDUPLICATE[]; + + /// "changelog-producer.row-deduplicate-ignore-fields" - Comma-separated fields to ignore when + /// comparing rows for changelog deduplication. This option is only valid when + /// "changelog-producer.row-deduplicate" is "true". + static const char CHANGELOG_PRODUCER_ROW_DEDUPLICATE_IGNORE_FIELDS[]; + + /// "changelog-file.prefix" - Specify the file name prefix of changelog files. Default value is + /// "changelog-". + static const char CHANGELOG_FILE_PREFIX[]; + + /// "changelog-file.format" - Specify the file format of changelog files. No default value. + static const char CHANGELOG_FILE_FORMAT[]; + + /// "changelog-file.compression" - Specify the compression of changelog files. No default + /// value. + static const char CHANGELOG_FILE_COMPRESSION[]; + /// "force-lookup" - Whether to force the use of lookup for compaction. Default value is /// "false". static const char FORCE_LOOKUP[]; diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 3de2b667e..858379651 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -288,6 +288,7 @@ set(PAIMON_CORE_SRCS core/io/vector_file_batch_reader.cpp core/io/file_index_evaluator.cpp core/io/key_value_data_file_record_reader.cpp + core/io/key_value_data_file_writer_factories.cpp core/io/key_value_data_file_writer_factory.cpp core/io/key_value_data_file_writer.cpp core/io/key_value_in_memory_record_reader.cpp @@ -814,6 +815,7 @@ if(PAIMON_BUILD_TESTS) core/mergetree/compact/deduplicate_merge_function_test.cpp core/mergetree/compact/first_row_merge_function_test.cpp core/mergetree/compact/first_row_merge_function_wrapper_test.cpp + core/mergetree/compact/internal_row_equalizer_test.cpp core/mergetree/compact/interval_partition_test.cpp core/mergetree/compact/lookup_changelog_merge_function_wrapper_test.cpp core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp index 39e6d494c..f0b0f5611 100644 --- a/src/paimon/common/defs.cpp +++ b/src/paimon/common/defs.cpp @@ -96,6 +96,12 @@ const char Options::DELETION_VECTOR_INDEX_FILE_TARGET_SIZE[] = "deletion-vector.index-file.target-size"; const char Options::DELETION_VECTOR_BITMAP64[] = "deletion-vectors.bitmap64"; const char Options::CHANGELOG_PRODUCER[] = "changelog-producer"; +const char Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE[] = "changelog-producer.row-deduplicate"; +const char Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE_IGNORE_FIELDS[] = + "changelog-producer.row-deduplicate-ignore-fields"; +const char Options::CHANGELOG_FILE_PREFIX[] = "changelog-file.prefix"; +const char Options::CHANGELOG_FILE_FORMAT[] = "changelog-file.format"; +const char Options::CHANGELOG_FILE_COMPRESSION[] = "changelog-file.compression"; const char Options::FORCE_LOOKUP[] = "force-lookup"; const char Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE[] = "partial-update.remove-record-on-delete"; diff --git a/src/paimon/common/utils/fields_comparator.cpp b/src/paimon/common/utils/fields_comparator.cpp index 224ec131f..2fee9aa15 100644 --- a/src/paimon/common/utils/fields_comparator.cpp +++ b/src/paimon/common/utils/fields_comparator.cpp @@ -85,28 +85,28 @@ Result FieldsComparator::CompareField( arrow::Type::type type = input_type->id(); switch (type) { case arrow::Type::type::BOOL: - return FieldsComparator::FieldComparatorFunc( + return FieldComparatorFunc( [field_idx](const InternalRow& lhs, const InternalRow& rhs) -> int32_t { bool lvalue = lhs.GetBoolean(field_idx); bool rvalue = rhs.GetBoolean(field_idx); return lvalue == rvalue ? 0 : (lvalue < rvalue ? -1 : 1); }); case arrow::Type::type::INT8: - return FieldsComparator::FieldComparatorFunc( + return FieldComparatorFunc( [field_idx](const InternalRow& lhs, const InternalRow& rhs) -> int32_t { int8_t lvalue = lhs.GetByte(field_idx); int8_t rvalue = rhs.GetByte(field_idx); return lvalue == rvalue ? 0 : (lvalue < rvalue ? -1 : 1); }); case arrow::Type::type::INT16: - return FieldsComparator::FieldComparatorFunc( + return FieldComparatorFunc( [field_idx](const InternalRow& lhs, const InternalRow& rhs) -> int32_t { int16_t lvalue = lhs.GetShort(field_idx); int16_t rvalue = rhs.GetShort(field_idx); return lvalue == rvalue ? 0 : (lvalue < rvalue ? -1 : 1); }); case arrow::Type::type::DATE32: - return FieldsComparator::FieldComparatorFunc( + return FieldComparatorFunc( [field_idx](const InternalRow& lhs, const InternalRow& rhs) -> int32_t { int32_t lvalue = lhs.GetDate(field_idx); int32_t rvalue = rhs.GetDate(field_idx); @@ -114,39 +114,36 @@ Result FieldsComparator::CompareField( }); case arrow::Type::type::INT32: - return FieldsComparator::FieldComparatorFunc( + return FieldComparatorFunc( [field_idx](const InternalRow& lhs, const InternalRow& rhs) -> int32_t { int32_t lvalue = lhs.GetInt(field_idx); int32_t rvalue = rhs.GetInt(field_idx); return lvalue == rvalue ? 0 : (lvalue < rvalue ? -1 : 1); }); case arrow::Type::type::INT64: - return FieldsComparator::FieldComparatorFunc( + return FieldComparatorFunc( [field_idx](const InternalRow& lhs, const InternalRow& rhs) -> int32_t { int64_t lvalue = lhs.GetLong(field_idx); int64_t rvalue = rhs.GetLong(field_idx); return lvalue == rvalue ? 0 : (lvalue < rvalue ? -1 : 1); }); case arrow::Type::type::FLOAT: - // TODO(xinyu.lxy): - // currently in java KeyComparatorSupplier: -inf < -0.0 == +0.0 < +inf = nan - // paimon-cpp: -inf < -0.0 == +0.0 < +inf and nan cannot be compared - return FieldsComparator::FieldComparatorFunc( + return FieldComparatorFunc( [field_idx](const InternalRow& lhs, const InternalRow& rhs) -> int32_t { float lvalue = lhs.GetFloat(field_idx); float rvalue = rhs.GetFloat(field_idx); - return lvalue == rvalue ? 0 : (lvalue < rvalue ? -1 : 1); + return CompareFloatingPoint(lvalue, rvalue); }); case arrow::Type::type::DOUBLE: - return FieldsComparator::FieldComparatorFunc( + return FieldComparatorFunc( [field_idx](const InternalRow& lhs, const InternalRow& rhs) -> int32_t { double lvalue = lhs.GetDouble(field_idx); double rvalue = rhs.GetDouble(field_idx); - return lvalue == rvalue ? 0 : (lvalue < rvalue ? -1 : 1); + return CompareFloatingPoint(lvalue, rvalue); }); case arrow::Type::type::STRING: case arrow::Type::type::BINARY: { - return FieldsComparator::FieldComparatorFunc( + return FieldComparatorFunc( [field_idx](const InternalRow& lhs, const InternalRow& rhs) -> int32_t { auto lvalue = lhs.GetStringView(field_idx); auto rvalue = rhs.GetStringView(field_idx); @@ -157,7 +154,7 @@ Result FieldsComparator::CompareField( case arrow::Type::type::TIMESTAMP: { auto timestamp_type = checked_pointer_cast(input_type); int32_t precision = DateTimeUtils::GetPrecisionFromType(timestamp_type); - return FieldsComparator::FieldComparatorFunc( + return FieldComparatorFunc( [field_idx, precision](const InternalRow& lhs, const InternalRow& rhs) -> int32_t { Timestamp lvalue = lhs.GetTimestamp(field_idx, precision); Timestamp rvalue = rhs.GetTimestamp(field_idx, precision); @@ -168,7 +165,7 @@ Result FieldsComparator::CompareField( auto* decimal_type = checked_cast(input_type.get()); auto precision = decimal_type->precision(); auto scale = decimal_type->scale(); - return FieldsComparator::FieldComparatorFunc( + return FieldComparatorFunc( [field_idx, precision, scale](const InternalRow& lhs, const InternalRow& rhs) -> int32_t { Decimal lvalue = lhs.GetDecimal(field_idx, precision, scale); diff --git a/src/paimon/common/utils/fields_comparator.h b/src/paimon/common/utils/fields_comparator.h index 7dbeca15e..50e0260d9 100644 --- a/src/paimon/common/utils/fields_comparator.h +++ b/src/paimon/common/utils/fields_comparator.h @@ -41,6 +41,9 @@ class DataField; /// A `Comparator` that compares the file store key. class FieldsComparator { public: + using FieldComparatorFunc = + std::function; + static Result> Create( const std::vector& input_data_field, bool is_ascending_order); @@ -82,9 +85,6 @@ class FieldsComparator { } private: - using FieldComparatorFunc = - std::function; - FieldsComparator(bool is_ascending_order, const std::vector& sort_fields, std::vector&& comparators) : is_ascending_order_(is_ascending_order), diff --git a/src/paimon/common/utils/fields_comparator_test.cpp b/src/paimon/common/utils/fields_comparator_test.cpp index 2f64f813c..111de404c 100644 --- a/src/paimon/common/utils/fields_comparator_test.cpp +++ b/src/paimon/common/utils/fields_comparator_test.cpp @@ -20,8 +20,10 @@ #include "paimon/common/utils/fields_comparator.h" #include +#include #include #include +#include #include "arrow/api.h" #include "gtest/gtest.h" @@ -80,6 +82,44 @@ class FieldsComparatorTest : public ::testing::Test { } CheckResult(row1, row2, input_types, sort_fields, has_null); } + + template + void CheckFloatingPointOrder(const std::shared_ptr& type) { + std::shared_ptr pool = GetDefaultPool(); + std::vector data_fields = {DataField(0, arrow::field("f0", type))}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr ascending_comparator, + FieldsComparator::Create(data_fields, + /*is_ascending_order=*/true)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr descending_comparator, + FieldsComparator::Create(data_fields, + /*is_ascending_order=*/false)); + + const T nan = std::numeric_limits::quiet_NaN(); + const std::vector values = {-std::numeric_limits::infinity(), + static_cast(-1), + static_cast(-0.0), + static_cast(0.0), + static_cast(1), + std::numeric_limits::infinity(), + nan}; + std::vector rows; + rows.reserve(values.size()); + for (T value : values) { + rows.emplace_back(BinaryRowGenerator::GenerateRow({value}, pool.get())); + } + + for (size_t i = 0; i < rows.size(); ++i) { + for (size_t j = 0; j < rows.size(); ++j) { + int32_t expected = i == j ? 0 : (i < j ? -1 : 1); + ASSERT_EQ(expected, ascending_comparator->CompareTo(rows[i], rows[j])); + ASSERT_EQ(-expected, descending_comparator->CompareTo(rows[i], rows[j])); + } + } + + BinaryRow negative_nan_row = BinaryRowGenerator::GenerateRow({-nan}, pool.get()); + ASSERT_EQ(0, ascending_comparator->CompareTo(rows.back(), negative_nan_row)); + ASSERT_EQ(0, ascending_comparator->CompareTo(negative_nan_row, rows.back())); + } }; TEST_F(FieldsComparatorTest, TestSimple) { @@ -202,6 +242,11 @@ TEST_F(FieldsComparatorTest, TestSimple) { } } +TEST_F(FieldsComparatorTest, TestFloatingPointOrder) { + CheckFloatingPointOrder(arrow::float32()); + CheckFloatingPointOrder(arrow::float64()); +} + TEST_F(FieldsComparatorTest, TestTimestampType) { auto pool = GetDefaultPool(); // test ts with different precision diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp index dfa0aa7ff..71ba73deb 100644 --- a/src/paimon/core/core_options.cpp +++ b/src/paimon/core/core_options.cpp @@ -330,9 +330,7 @@ class ConfigParser { // storing various configurable fields and their default values. struct CoreOptions::Impl { int64_t page_size = 64 * 1024; - std::optional target_file_size; int64_t target_file_row_num = std::numeric_limits::max(); - std::optional blob_target_file_size; int64_t source_split_target_size = 128 * 1024 * 1024; int64_t source_split_open_file_cost = 4 * 1024 * 1024; int64_t manifest_target_file_size = 8 * 1024 * 1024; @@ -343,20 +341,33 @@ struct CoreOptions::Impl { int64_t commit_timeout = std::numeric_limits::max(); int64_t commit_min_retry_wait = 10; int64_t commit_max_retry_wait = 10 * 1000; - bool realtime_enabled = false; int64_t realtime_read_view_ttl_millis = 5 * 60 * 1000; - StatisticsMode realtime_store_statistics_mode = StatisticsMode::NONE; + int64_t write_buffer_spill_max_disk_size = std::numeric_limits::max(); + double variant_shredding_min_field_cardinality_ratio = 0.1; + double variant_shredding_adaptive_retention_ratio = 0.05; + double lookup_cache_bloom_filter_fpp = 0.05; + int64_t cache_page_size = 64 * 1024; // 64KB + int64_t lookup_cache_max_memory = 256 * 1024 * 1024; + double lookup_cache_high_prio_pool_ratio = 0.25; + int64_t lookup_cache_file_retention_ms = 1 * 3600 * 1000; // 1 hour + int64_t lookup_cache_max_disk_size = INT64_MAX; + std::optional target_file_size; + std::optional blob_target_file_size; std::shared_ptr file_format; std::shared_ptr file_system; std::shared_ptr manifest_file_format; std::shared_ptr cache; - std::optional scan_snapshot_id; std::optional scan_timestamp_millis; + std::optional optimized_compaction_interval; + std::optional compaction_total_size_threshold; + std::optional compaction_incremental_size_threshold; + std::shared_ptr changelog_file_format; ExpireConfig expire_config; std::vector sequence_field; std::vector remove_record_on_sequence_group; + std::vector changelog_row_deduplicate_ignore_fields; std::vector blob_fields; std::vector blob_descriptor_fields; std::vector blob_view_fields; @@ -367,20 +378,25 @@ struct CoreOptions::Impl { std::string manifest_compression = "zstd"; std::string branch = BranchManager::DEFAULT_MAIN_BRANCH; std::string data_file_prefix = "data-"; + std::string changelog_file_prefix = "changelog-"; std::string file_system_scheme_to_identifier_map_str; - std::optional field_default_func; std::optional scan_fallback_branch; std::optional data_file_external_paths; std::optional blob_view_upstream_warehouse; - + std::optional changelog_file_compression; + std::optional global_index_external_path; + std::optional scan_tag_name; + CompressOptions lookup_compress_options{"zstd", 1}; + CompressOptions spill_compress_options{"zstd", 1}; std::map raw_options; + std::map> file_format_per_level; + std::map file_compression_per_level; + StatisticsMode realtime_store_statistics_mode = StatisticsMode::NONE; int32_t bucket = -1; - int32_t manifest_merge_min_count = 30; int32_t scan_manifest_entry_cache_max_snapshots = 0; - bool scan_manifest_entry_lazy_decode_enabled = true; int32_t read_batch_size = 1024; int32_t write_batch_size = 1024; int32_t local_sort_max_num_file_handles = 128; @@ -389,32 +405,43 @@ struct CoreOptions::Impl { int32_t compaction_max_size_amplification_percent = 200; int32_t compaction_size_ratio = 1; int32_t num_sorted_runs_compaction_trigger = 5; - std::optional num_sorted_runs_stop_trigger; - std::optional num_levels; - SortOrder sequence_field_sort_order = SortOrder::ASCENDING; MergeEngine merge_engine = MergeEngine::DEDUPLICATE; SortEngine sort_engine = SortEngine::LOSER_TREE; ChangelogProducer changelog_producer = ChangelogProducer::NONE; ExternalPathStrategy external_path_strategy = ExternalPathStrategy::NONE; LookupCompactMode lookup_compact_mode = LookupCompactMode::RADICAL; - std::optional lookup_compact_max_interval; BucketFunctionType bucket_function_type = BucketFunctionType::DEFAULT; - int32_t file_compression_zstd_level = 1; - int64_t write_buffer_spill_max_disk_size = std::numeric_limits::max(); + CoreOptions::SequenceNumberInitMode write_sequence_number_init_mode = + CoreOptions::SequenceNumberInitMode::SCAN; + VariantShreddingInferenceMode variant_shredding_inference_mode = + VariantShreddingInferenceMode::PER_FILE; + int32_t variant_shredding_max_schema_width = 300; + int32_t variant_shredding_max_schema_depth = 50; + int32_t variant_shredding_max_infer_buffer_row = 4096; + int32_t variant_shredding_adaptive_max_infer_buffer_row = 256; + int32_t compact_off_peak_start_hour = -1; + int32_t compact_off_peak_end_hour = -1; + int32_t compact_off_peak_ratio = 0; + int32_t lookup_remote_level_threshold = INT32_MIN; + std::optional num_sorted_runs_stop_trigger; + std::optional num_levels; + std::optional lookup_compact_max_interval; + std::optional global_index_thread_num; + bool realtime_enabled = false; + bool scan_manifest_entry_lazy_decode_enabled = true; bool ignore_delete = false; bool manifest_delete_file_drop_stats = false; bool write_buffer_spillable = true; bool write_only = false; bool bucket_append_ordered = false; - CoreOptions::SequenceNumberInitMode write_sequence_number_init_mode = - CoreOptions::SequenceNumberInitMode::SCAN; bool deletion_vectors_enabled = false; bool deletion_vectors_bitmap64 = false; bool force_lookup = false; bool lookup_wait = true; + bool changelog_row_deduplicate = false; bool partial_update_remove_record_on_delete = false; bool aggregation_remove_record_on_delete = false; bool table_read_sequence_number_enabled = false; @@ -427,48 +454,19 @@ struct CoreOptions::Impl { bool row_tracking_partition_group_on_commit = true; bool data_evolution_enabled = false; bool variant_infer_shredding_schema = false; - VariantShreddingInferenceMode variant_shredding_inference_mode = - VariantShreddingInferenceMode::PER_FILE; - int32_t variant_shredding_max_schema_width = 300; - int32_t variant_shredding_max_schema_depth = 50; - double variant_shredding_min_field_cardinality_ratio = 0.1; - int32_t variant_shredding_max_infer_buffer_row = 4096; - int32_t variant_shredding_adaptive_max_infer_buffer_row = 256; - double variant_shredding_adaptive_retention_ratio = 0.05; bool blob_view_resolve_enabled = true; bool blob_as_descriptor = false; - std::optional blob_split_by_file_size; bool legacy_partition_name_enabled = true; bool global_index_enabled = true; - std::optional global_index_thread_num; bool commit_force_compact = false; bool commit_discard_duplicate_files = false; bool dynamic_partition_overwrite = true; bool overwrite_upgrade = true; bool compaction_force_rewrite_all_files = false; bool compaction_force_up_level_0 = false; - std::optional global_index_external_path; - - std::optional scan_tag_name; - std::optional optimized_compaction_interval; - std::optional compaction_total_size_threshold; - std::optional compaction_incremental_size_threshold; - int32_t compact_off_peak_start_hour = -1; - int32_t compact_off_peak_end_hour = -1; - int32_t compact_off_peak_ratio = 0; bool lookup_cache_bloom_filter = true; - double lookup_cache_bloom_filter_fpp = 0.05; bool lookup_remote_file_enabled = false; - int32_t lookup_remote_level_threshold = INT32_MIN; - CompressOptions lookup_compress_options{"zstd", 1}; - CompressOptions spill_compress_options{"zstd", 1}; - int64_t cache_page_size = 64 * 1024; // 64KB - std::map> file_format_per_level; - std::map file_compression_per_level; - int64_t lookup_cache_max_memory = 256 * 1024 * 1024; - double lookup_cache_high_prio_pool_ratio = 0.25; - int64_t lookup_cache_file_retention_ms = 1 * 3600 * 1000; // 1 hour - int64_t lookup_cache_max_disk_size = INT64_MAX; + std::optional blob_split_by_file_size; // Parse basic table options: bucket, partition, file sizes, batch sizes, file system, etc. Status ParseBasicOptions( @@ -542,6 +540,8 @@ struct CoreOptions::Impl { PAIMON_RETURN_NOT_OK(parser.ParseExternalPathStrategy(&external_path_strategy)); // Parse data-file.prefix - file name prefix of data files, default "data-" PAIMON_RETURN_NOT_OK(parser.Parse(Options::DATA_FILE_PREFIX, &data_file_prefix)); + // Parse changelog-file.prefix - file name prefix of changelog files, default "changelog-" + PAIMON_RETURN_NOT_OK(parser.Parse(Options::CHANGELOG_FILE_PREFIX, &changelog_file_prefix)); // Parse row-tracking.enabled - whether to enable unique row id for append table PAIMON_RETURN_NOT_OK( parser.Parse(Options::ROW_TRACKING_ENABLED, &row_tracking_enabled)); @@ -590,6 +590,14 @@ struct CoreOptions::Impl { Options::FILE_FORMAT, /*default_identifier=*/"parquet", &file_format)); // Parse file.compression - default file compression, default "zstd" PAIMON_RETURN_NOT_OK(parser.Parse(Options::FILE_COMPRESSION, &file_compression)); + // Parse changelog-file.format - no default value + if (parser.ContainsKey(Options::CHANGELOG_FILE_FORMAT)) { + PAIMON_RETURN_NOT_OK(parser.ParseObject( + Options::CHANGELOG_FILE_FORMAT, file_format->Identifier(), &changelog_file_format)); + } + // Parse changelog-file.compression - no default value + PAIMON_RETURN_NOT_OK( + parser.Parse(Options::CHANGELOG_FILE_COMPRESSION, &changelog_file_compression)); // Parse file.compression.zstd-level - zstd compression level, default 1 PAIMON_RETURN_NOT_OK( parser.Parse(Options::FILE_COMPRESSION_ZSTD_LEVEL, &file_compression_zstd_level)); @@ -720,6 +728,13 @@ struct CoreOptions::Impl { PAIMON_RETURN_NOT_OK(parser.Parse(Options::FIELDS_DEFAULT_AGG_FUNC, &field_default_func)); // Parse changelog-producer - whether to double write to a changelog file, default "none" PAIMON_RETURN_NOT_OK(parser.ParseChangelogProducer(&changelog_producer)); + // Parse changelog-producer.row-deduplicate - skip unchanged row changelogs + PAIMON_RETURN_NOT_OK(parser.Parse(Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE, + &changelog_row_deduplicate)); + // Parse changelog-producer.row-deduplicate-ignore-fields - ignored comparison fields + PAIMON_RETURN_NOT_OK(parser.ParseList( + Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE_IGNORE_FIELDS, Options::FIELDS_SEPARATOR, + &changelog_row_deduplicate_ignore_fields, /*need_trim=*/true)); // Parse partial-update.remove-record-on-delete - remove whole row on delete PAIMON_RETURN_NOT_OK(parser.Parse(Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, &partial_update_remove_record_on_delete)); @@ -1578,6 +1593,26 @@ ChangelogProducer CoreOptions::GetChangelogProducer() const { return impl_->changelog_producer; } +bool CoreOptions::ChangelogRowDeduplicate() const { + return impl_->changelog_row_deduplicate; +} + +const std::vector& CoreOptions::GetChangelogRowDeduplicateIgnoreFields() const { + return impl_->changelog_row_deduplicate_ignore_fields; +} + +std::string CoreOptions::ChangelogFilePrefix() const { + return impl_->changelog_file_prefix; +} + +std::shared_ptr CoreOptions::GetChangelogFileFormat() const { + return impl_->changelog_file_format; +} + +std::optional CoreOptions::GetChangelogFileCompression() const { + return impl_->changelog_file_compression; +} + LookupStrategy CoreOptions::GetLookupStrategy() const { return LookupStrategy::From( /*is_first_row=*/GetMergeEngine() == MergeEngine::FIRST_ROW, diff --git a/src/paimon/core/core_options.h b/src/paimon/core/core_options.h index f4e7964f7..345958e1a 100644 --- a/src/paimon/core/core_options.h +++ b/src/paimon/core/core_options.h @@ -207,6 +207,11 @@ class PAIMON_EXPORT CoreOptions { bool DeletionVectorsBitmap64() const; int64_t DeletionVectorTargetFileSize() const; ChangelogProducer GetChangelogProducer() const; + bool ChangelogRowDeduplicate() const; + const std::vector& GetChangelogRowDeduplicateIgnoreFields() const; + std::string ChangelogFilePrefix() const; + std::shared_ptr GetChangelogFileFormat() const; + std::optional GetChangelogFileCompression() const; LookupStrategy GetLookupStrategy() const; bool NeedLookup() const; diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp index b0a3274ba..c424b9cfe 100644 --- a/src/paimon/core/core_options_test.cpp +++ b/src/paimon/core/core_options_test.cpp @@ -39,6 +39,7 @@ TEST(CoreOptionsTest, TestDefaultValue) { ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({})); ASSERT_EQ(core_options.GetManifestFormat()->Identifier(), "avro"); ASSERT_EQ(core_options.GetFileFormat()->Identifier(), "parquet"); + ASSERT_EQ(nullptr, core_options.GetChangelogFileFormat()); ASSERT_EQ(core_options.GetWriteFileFormat(0)->Identifier(), "parquet"); ASSERT_EQ(core_options.GetWriteFileFormat(3)->Identifier(), "parquet"); ASSERT_TRUE(core_options.GetFileSystem()); @@ -58,6 +59,7 @@ TEST(CoreOptionsTest, TestDefaultValue) { ASSERT_FALSE(core_options.RealtimeEnabled()); ASSERT_EQ(StatisticsMode::NONE, core_options.GetRealtimeStoreStatisticsMode()); ASSERT_EQ("zstd", core_options.GetFileCompression()); + ASSERT_EQ(std::nullopt, core_options.GetChangelogFileCompression()); ASSERT_EQ("zstd", core_options.GetWriteFileCompression(0)); ASSERT_EQ("zstd", core_options.GetWriteFileCompression(3)); ASSERT_EQ("zstd", core_options.GetManifestCompression()); @@ -125,6 +127,9 @@ TEST(CoreOptionsTest, TestDefaultValue) { ASSERT_FALSE(core_options.DeletionVectorsBitmap64()); ASSERT_EQ(2 * 1024 * 1024, core_options.DeletionVectorTargetFileSize()); ASSERT_EQ(ChangelogProducer::NONE, core_options.GetChangelogProducer()); + ASSERT_FALSE(core_options.ChangelogRowDeduplicate()); + ASSERT_TRUE(core_options.GetChangelogRowDeduplicateIgnoreFields().empty()); + ASSERT_EQ("changelog-", core_options.ChangelogFilePrefix()); ASSERT_FALSE(core_options.NeedLookup()); ASSERT_FALSE(core_options.PrepareCommitWaitCompaction()); LookupStrategy expected_lookup_strategy = {/*is_first_row=*/false, @@ -192,6 +197,7 @@ TEST(CoreOptionsTest, TestFromMap) { std::map options = { {Options::FILE_SYSTEM, "Local"}, {Options::FILE_FORMAT, "ORC"}, + {Options::CHANGELOG_FILE_FORMAT, "avro"}, {Options::MANIFEST_FORMAT, "avRo"}, {Options::BUCKET, "3"}, {Options::PAGE_SIZE, "128 kb"}, @@ -248,6 +254,10 @@ TEST(CoreOptionsTest, TestFromMap) { {Options::DELETION_VECTOR_BITMAP64, "true"}, {Options::DELETION_VECTOR_INDEX_FILE_TARGET_SIZE, "4MB"}, {Options::CHANGELOG_PRODUCER, "full-compaction"}, + {Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE, "true"}, + {Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE_IGNORE_FIELDS, "f0, f2"}, + {Options::CHANGELOG_FILE_PREFIX, "test-changelog-"}, + {Options::CHANGELOG_FILE_COMPRESSION, "lz4"}, {Options::FORCE_LOOKUP, "true"}, {"fields.g_1,g_3.sequence-group", "c,d"}, {Options::AGGREGATION_REMOVE_RECORD_ON_DELETE, "true"}, @@ -324,6 +334,7 @@ TEST(CoreOptionsTest, TestFromMap) { ASSERT_TRUE(fs); ASSERT_EQ(core_options.GetFileFormat()->Identifier(), "orc"); + ASSERT_EQ(core_options.GetChangelogFileFormat()->Identifier(), "avro"); ASSERT_EQ(core_options.GetWriteFileFormat(0)->Identifier(), "avro"); ASSERT_EQ(core_options.GetWriteFileFormat(1)->Identifier(), "orc"); ASSERT_EQ(core_options.GetWriteFileFormat(3)->Identifier(), "parquet"); @@ -393,6 +404,11 @@ TEST(CoreOptionsTest, TestFromMap) { ASSERT_TRUE(core_options.DeletionVectorsBitmap64()); ASSERT_EQ(4 * 1024 * 1024, core_options.DeletionVectorTargetFileSize()); ASSERT_EQ(ChangelogProducer::FULL_COMPACTION, core_options.GetChangelogProducer()); + ASSERT_TRUE(core_options.ChangelogRowDeduplicate()); + ASSERT_EQ(std::vector({"f0", "f2"}), + core_options.GetChangelogRowDeduplicateIgnoreFields()); + ASSERT_EQ("test-changelog-", core_options.ChangelogFilePrefix()); + ASSERT_EQ(std::optional("lz4"), core_options.GetChangelogFileCompression()); ASSERT_TRUE(core_options.NeedLookup()); ASSERT_TRUE(core_options.PrepareCommitWaitCompaction()); LookupStrategy expected_lookup_strategy = {/*is_first_row=*/false, diff --git a/src/paimon/core/io/async_key_value_producer_and_consumer.cpp b/src/paimon/core/io/async_key_value_producer_and_consumer.cpp index 1792b43cd..b353aefc8 100644 --- a/src/paimon/core/io/async_key_value_producer_and_consumer.cpp +++ b/src/paimon/core/io/async_key_value_producer_and_consumer.cpp @@ -25,22 +25,96 @@ #include "arrow/c/abi.h" #include "arrow/c/helpers.h" +#include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/reader/reader_utils.h" #include "paimon/reader/batch_reader.h" namespace paimon { class MemoryPool; +namespace { + +class AsyncKeyValueQueueBatchSink : public AsyncKeyValueBatchSink { + public: + AsyncKeyValueQueueBatchSink(std::atomic* consume_finished, + tbb::concurrent_bounded_queue* kv_queue) + : consume_finished_(consume_finished), kv_queue_(kv_queue) {} + + Status Write(AsyncKeyValueBatchType type, std::vector&& rows) override { + if (*consume_finished_) { + return Status::Cancelled("Key value conversion is cancelled"); + } + kv_queue_->push(AsyncKeyValueRowsBatch{type, std::move(rows)}); + return Status::OK(); + } + + bool IsCancelled() const override { + return *consume_finished_; + } + + private: + std::atomic* consume_finished_; + tbb::concurrent_bounded_queue* kv_queue_; +}; + +} // namespace + +std::shared_ptr AsyncKeyValueBatchProducer::GetReaderMetrics() const { + return std::make_shared(); +} + +SortMergeReaderBatchProducer::SortMergeReaderBatchProducer( + std::unique_ptr&& sort_merge_reader, int32_t batch_size) + : sort_merge_reader_(std::move(sort_merge_reader)), + batch_size_(NormalizeProjectionBatchSize(batch_size)) {} + +Status SortMergeReaderBatchProducer::Produce(AsyncKeyValueBatchSink* sink) { + std::vector batch; + batch.reserve(batch_size_); + while (!sink->IsCancelled()) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr iterator, + sort_merge_reader_->NextBatch()); + if (iterator == nullptr) { + break; + } + while (!sink->IsCancelled()) { + PAIMON_ASSIGN_OR_RAISE(bool has_next, iterator->HasNext()); + if (!has_next) { + break; + } + batch.push_back(std::move(iterator->Next())); + if (static_cast(batch.size()) >= batch_size_) { + PAIMON_RETURN_NOT_OK(sink->Write(AsyncKeyValueBatchType::DATA, std::move(batch))); + batch = std::vector(); + batch.reserve(batch_size_); + } + } + } + if (!batch.empty() && !sink->IsCancelled()) { + PAIMON_RETURN_NOT_OK(sink->Write(AsyncKeyValueBatchType::DATA, std::move(batch))); + } + return Status::OK(); +} + +std::shared_ptr SortMergeReaderBatchProducer::GetReaderMetrics() const { + return sort_merge_reader_->GetReaderMetrics(); +} + +void SortMergeReaderBatchProducer::Close() { + if (!closed_) { + sort_merge_reader_->Close(); + closed_ = true; + } +} + template AsyncKeyValueProducerAndConsumer::AsyncKeyValueProducerAndConsumer( - std::unique_ptr&& sort_merge_reader, ConsumerCreator create_consumer, - int32_t batch_size, int32_t consumer_thread_num, const std::shared_ptr& pool) - : batch_size_(std::min(batch_size, MAX_PROJECTION_BATCH_SIZE)), - consumer_thread_num_(consumer_thread_num), - pool_(pool), - sort_merge_reader_(std::move(sort_merge_reader)), - create_consumer_(std::move(create_consumer)) { - kv_queue_.set_capacity(consumer_thread_num * 2); + std::unique_ptr&& producer, ConsumerCreator create_consumer, + int32_t consumer_thread_num) + : consumer_thread_num_(consumer_thread_num), + create_consumer_(std::move(create_consumer)), + producer_(std::move(producer)) { + kv_queue_.set_capacity(consumer_thread_num_ * 2); result_queue_.set_capacity(RESULT_BATCH_COUNT); } @@ -70,6 +144,12 @@ Status AsyncKeyValueProducerAndConsumer::CheckStatusAndCleanUp() { template Result AsyncKeyValueProducerAndConsumer::NextBatch() { + PAIMON_ASSIGN_OR_RAISE(AsyncKeyValueResultBatch result, NextBatchWithType()); + return std::move(result.result); +} + +template +Result> AsyncKeyValueProducerAndConsumer::NextBatchWithType() { if (!producer_future_.valid()) { producer_future_ = std::async(std::launch::async, &AsyncKeyValueProducerAndConsumer::ProduceLoop, this) @@ -78,10 +158,10 @@ Result AsyncKeyValueProducerAndConsumer::NextBatch() { if (consumers_.empty()) { consumers_.reserve(consumer_thread_num_); for (int32_t i = 0; i < consumer_thread_num_; i++) { - Result>> consumer = create_consumer_(); - PAIMON_RETURN_NOT_OK(consumer.status()); + std::unique_ptr> consumer; + PAIMON_ASSIGN_OR_RAISE(consumer, create_consumer_()); auto async_consumer = std::make_unique>( - std::move(consumer).value(), consume_finished_, consumer_finished_count_, kv_queue_, + std::move(consumer), consume_finished_, consumer_finished_count_, kv_queue_, result_queue_); consumers_.push_back(std::move(async_consumer)); } @@ -90,16 +170,16 @@ Result AsyncKeyValueProducerAndConsumer::NextBatch() { if (next_batch_finished_) { // projection reader is eof - return R(); + return AsyncKeyValueResultBatch(); } - R result; + AsyncKeyValueResultBatch result; while (!result_queue_.try_pop(result)) { PAIMON_RETURN_NOT_OK(CheckStatusAndCleanUp()); if (consumer_finished_count_ == consumer_thread_num_ && result_queue_.empty()) { // all consume thread finished next_batch_finished_ = true; - return R(); + return AsyncKeyValueResultBatch(); } usleep(1000); } @@ -109,33 +189,9 @@ Result AsyncKeyValueProducerAndConsumer::NextBatch() { template Status AsyncKeyValueProducerAndConsumer::ProduceLoop() { - std::vector batch; - batch.reserve(batch_size_); - while (!consume_finished_) { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr iterator, - sort_merge_reader_->NextBatch()); - if (iterator == nullptr) { - break; - } - while (!consume_finished_) { - PAIMON_ASSIGN_OR_RAISE(bool has_next, iterator->HasNext()); - if (!has_next) { - break; - } - batch.push_back(std::move(iterator->Next())); - if (static_cast(batch.size()) >= batch_size_) { - kv_queue_.push(std::move(batch)); - batch = std::vector(); - batch.reserve(batch_size_); - } - } - } - // Push remaining rows - if (!batch.empty()) { - kv_queue_.push(std::move(batch)); - } - // Push empty batch as EOF signal - kv_queue_.push(std::vector()); + AsyncKeyValueQueueBatchSink sink(&consume_finished_, &kv_queue_); + PAIMON_RETURN_NOT_OK(producer_->Produce(&sink)); + kv_queue_.push(AsyncKeyValueRowsBatch()); return Status::OK(); } @@ -155,8 +211,9 @@ void AsyncKeyValueProducerAndConsumer::CleanUp() { template void AsyncKeyValueProducerAndConsumer::CleanUpQueue() { - R read_batch; - while (result_queue_.try_pop(read_batch)) { + AsyncKeyValueResultBatch tagged_batch; + while (result_queue_.try_pop(tagged_batch)) { + R& read_batch = tagged_batch.result; if constexpr (std::is_same_v) { if (!BatchReader::IsEofBatch(read_batch)) { ReaderUtils::ReleaseReadBatch(std::move(read_batch)); @@ -168,7 +225,7 @@ void AsyncKeyValueProducerAndConsumer::CleanUpQueue() { } } - std::vector kv_batch; + AsyncKeyValueRowsBatch kv_batch; while (kv_queue_.try_pop(kv_batch)) { } } @@ -178,11 +235,11 @@ template class AsyncKeyValueProducerAndConsumer; template AsyncKeyValueConsumer::AsyncKeyValueConsumer( - std::unique_ptr>&& key_value_consumer, - std::atomic& consume_finished, std::atomic& consumer_finished_count, - tbb::concurrent_bounded_queue>& kv_queue, - tbb::concurrent_bounded_queue& result_queue) - : key_value_consumer_(std::move(key_value_consumer)), + std::unique_ptr>&& consumer, std::atomic& consume_finished, + std::atomic& consumer_finished_count, + tbb::concurrent_bounded_queue& kv_queue, + tbb::concurrent_bounded_queue>& result_queue) + : consumer_(std::move(consumer)), consume_finished_(consume_finished), consumer_finished_count_(consumer_finished_count), kv_queue_(kv_queue), @@ -205,18 +262,18 @@ Status AsyncKeyValueConsumer::GetStatus() const { template Status AsyncKeyValueConsumer::ConsumeLoop() { while (!consume_finished_) { - std::vector key_value_vec; - if (!kv_queue_.try_pop(key_value_vec)) { + AsyncKeyValueRowsBatch rows_batch; + if (!kv_queue_.try_pop(rows_batch)) { usleep(100); continue; } - if (key_value_vec.empty()) { + if (rows_batch.rows.empty()) { // Empty batch is EOF signal; re-push for other consumers - kv_queue_.push(std::move(key_value_vec)); + kv_queue_.push(std::move(rows_batch)); break; } - PAIMON_ASSIGN_OR_RAISE(R result, key_value_consumer_->NextBatch(key_value_vec)); - result_queue_.push(std::move(result)); + PAIMON_ASSIGN_OR_RAISE(R result, consumer_->NextBatch(rows_batch.rows)); + result_queue_.push(AsyncKeyValueResultBatch{rows_batch.type, std::move(result)}); } consumer_finished_count_++; return Status::OK(); @@ -227,7 +284,9 @@ void AsyncKeyValueConsumer::CleanUp() { if (consumer_future_.valid()) { [[maybe_unused]] Status status = consumer_future_.get(); } - key_value_consumer_->CleanUp(); + if (consumer_) { + consumer_->CleanUp(); + } } template class AsyncKeyValueConsumer; diff --git a/src/paimon/core/io/async_key_value_producer_and_consumer.h b/src/paimon/core/io/async_key_value_producer_and_consumer.h index af8bbed68..086ff1112 100644 --- a/src/paimon/core/io/async_key_value_producer_and_consumer.h +++ b/src/paimon/core/io/async_key_value_producer_and_consumer.h @@ -18,6 +18,7 @@ #pragma once +#include #include #include #include @@ -37,21 +38,80 @@ namespace paimon { template class AsyncKeyValueConsumer; -class MemoryPool; class Metrics; -// Asynchronous iterate SortMergeReader (producer) and row-to-array conversion (consumer), support -// multi-threaded conversion, R can be BatchReader::ReadBatch, KeyValueBatch +enum class AsyncKeyValueBatchType { + DATA, + CHANGELOG, +}; + +struct AsyncKeyValueRowsBatch { + AsyncKeyValueBatchType type = AsyncKeyValueBatchType::DATA; + std::vector rows; +}; + +class AsyncKeyValueBatchSink { + public: + virtual ~AsyncKeyValueBatchSink() = default; + + virtual Status Write(AsyncKeyValueBatchType type, std::vector&& rows) = 0; + + virtual bool IsCancelled() const = 0; +}; + +class AsyncKeyValueBatchProducer { + public: + virtual ~AsyncKeyValueBatchProducer() = default; + + virtual Status Produce(AsyncKeyValueBatchSink* sink) = 0; + + virtual std::shared_ptr GetReaderMetrics() const; + + virtual void Close() {} + + protected: + // Limits the number of rows sent to one Arrow projection call. + static int32_t NormalizeProjectionBatchSize(int32_t batch_size) { + return std::min(batch_size, MAX_PROJECTION_BATCH_SIZE); + } + + private: + static constexpr int32_t MAX_PROJECTION_BATCH_SIZE = 100000; +}; + +class SortMergeReaderBatchProducer : public AsyncKeyValueBatchProducer { + public: + SortMergeReaderBatchProducer(std::unique_ptr&& sort_merge_reader, + int32_t batch_size); + + Status Produce(AsyncKeyValueBatchSink* sink) override; + + std::shared_ptr GetReaderMetrics() const override; + + void Close() override; + + private: + std::unique_ptr sort_merge_reader_; + int32_t batch_size_; + bool closed_ = false; +}; + +template +struct AsyncKeyValueResultBatch { + AsyncKeyValueBatchType type = AsyncKeyValueBatchType::DATA; + R result; +}; + +// Asynchronous iterates AsyncKeyValueBatchProducer(producer) and row-to-array conversion +// (consumer), support multi-threaded conversion, R can be BatchReader::ReadBatch or KeyValueBatch. template class AsyncKeyValueProducerAndConsumer { public: using ConsumerCreator = std::function>>()>; - AsyncKeyValueProducerAndConsumer(std::unique_ptr&& sort_merge_reader, - ConsumerCreator create_consumer, int32_t batch_size, - int32_t consumer_thread_num, - const std::shared_ptr& pool); + AsyncKeyValueProducerAndConsumer(std::unique_ptr&& producer, + ConsumerCreator create_consumer, int32_t consumer_thread_num); ~AsyncKeyValueProducerAndConsumer() { CleanUp(); @@ -59,21 +119,20 @@ class AsyncKeyValueProducerAndConsumer { Result NextBatch(); + Result> NextBatchWithType(); + std::shared_ptr GetReaderMetrics() const { - return sort_merge_reader_->GetReaderMetrics(); + return producer_->GetReaderMetrics(); } void Close() { CleanUp(); - sort_merge_reader_->Close(); + producer_->Close(); } private: static constexpr int32_t RESULT_BATCH_COUNT = 3; - // in case write batch size is too large and overflow arrow array - static constexpr int32_t MAX_PROJECTION_BATCH_SIZE = 100000; - void CleanUpQueue(); Status ProduceLoop(); void CleanUp(); @@ -81,11 +140,9 @@ class AsyncKeyValueProducerAndConsumer { Status CheckStatusAndCleanUp(); private: - int32_t batch_size_; int32_t consumer_thread_num_; - std::shared_ptr pool_; - std::unique_ptr sort_merge_reader_; ConsumerCreator create_consumer_; + std::unique_ptr producer_; // produce: merge sort KeyValue and push result KeyValue to kv_queue_, consume: project KeyValue // to arrow array and push result array to result_queue_ @@ -94,18 +151,18 @@ class AsyncKeyValueProducerAndConsumer { std::shared_future producer_future_; std::vector>> consumers_; std::atomic consumer_finished_count_ = 0; - tbb::concurrent_bounded_queue> kv_queue_; - tbb::concurrent_bounded_queue result_queue_; + tbb::concurrent_bounded_queue kv_queue_; + tbb::concurrent_bounded_queue> result_queue_; }; template class AsyncKeyValueConsumer { public: - AsyncKeyValueConsumer(std::unique_ptr>&& key_value_consumer, + AsyncKeyValueConsumer(std::unique_ptr>&& consumer, std::atomic& consume_finished, std::atomic& consumer_finished_count, - tbb::concurrent_bounded_queue>& kv_queue, - tbb::concurrent_bounded_queue& result_queue); + tbb::concurrent_bounded_queue& kv_queue, + tbb::concurrent_bounded_queue>& result_queue); ~AsyncKeyValueConsumer() { CleanUp(); @@ -118,12 +175,12 @@ class AsyncKeyValueConsumer { Status ConsumeLoop(); private: - std::unique_ptr> key_value_consumer_; + std::unique_ptr> consumer_; std::shared_future consumer_future_; std::atomic& consume_finished_; std::atomic& consumer_finished_count_; - tbb::concurrent_bounded_queue>& kv_queue_; - tbb::concurrent_bounded_queue& result_queue_; + tbb::concurrent_bounded_queue& kv_queue_; + tbb::concurrent_bounded_queue>& result_queue_; }; } // namespace paimon diff --git a/src/paimon/core/io/async_key_value_projection_reader.h b/src/paimon/core/io/async_key_value_projection_reader.h index a272a0edf..79c7aa4de 100644 --- a/src/paimon/core/io/async_key_value_projection_reader.h +++ b/src/paimon/core/io/async_key_value_projection_reader.h @@ -38,10 +38,12 @@ class AsyncKeyValueProjectionReader : public BatchReader { -> Result>> { return KeyValueProjectionConsumer::Create(target_schema, target_to_src_mapping, pool); }; + std::unique_ptr producer = + std::make_unique(std::move(sort_merge_reader), + batch_size); producer_and_consumer_ = std::make_unique>( - std::move(sort_merge_reader), create_consumer, batch_size, projection_thread_num, - pool); + std::move(producer), create_consumer, projection_thread_num); } Result NextBatch() override { diff --git a/src/paimon/core/io/data_file_path_factory.h b/src/paimon/core/io/data_file_path_factory.h index b49154f1b..a9050eeee 100644 --- a/src/paimon/core/io/data_file_path_factory.h +++ b/src/paimon/core/io/data_file_path_factory.h @@ -56,8 +56,9 @@ class DataFilePathFactory : public PathFactory { return NewPath(data_file_prefix_); } - std::string NewChangelogPath() const { - return NewPath(std::string(CHANGELOG_FILE_PREFIX)); + std::string NewChangelogPath(const std::string& changelog_file_prefix, + const std::string& format_identifier) const { + return NewPathFromName(NewFileName(changelog_file_prefix, "." + format_identifier)); } std::string NewBlobPath() const { diff --git a/src/paimon/core/io/data_file_path_factory_test.cpp b/src/paimon/core/io/data_file_path_factory_test.cpp index 6283876df..fe4824c94 100644 --- a/src/paimon/core/io/data_file_path_factory_test.cpp +++ b/src/paimon/core/io/data_file_path_factory_test.cpp @@ -59,6 +59,13 @@ TEST_F(DataFilePathFactoryTest, TestNewPath) { ASSERT_EQ(factory_.NewPathFromName("index-file"), "/tmp/index-file"); } +TEST_F(DataFilePathFactoryTest, TestNewChangelogPath) { + std::string path = factory_.NewChangelogPath("changes-", "parquet"); + + ASSERT_TRUE(path.find("/tmp/changes-") != std::string::npos); + ASSERT_TRUE(StringUtils::EndsWith(path, ".parquet")); +} + TEST_F(DataFilePathFactoryTest, TestNewPathWithDataFilePrefixAndExternalPath) { DataFilePathFactory factory; ASSERT_OK_AND_ASSIGN( diff --git a/src/paimon/core/io/key_value_data_file_writer_factories.cpp b/src/paimon/core/io/key_value_data_file_writer_factories.cpp new file mode 100644 index 000000000..5aa5a8caa --- /dev/null +++ b/src/paimon/core/io/key_value_data_file_writer_factories.cpp @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/io/key_value_data_file_writer_factories.h" + +#include "paimon/common/data/shredding/shredding_write_plan_factories.h" +#include "paimon/core/io/key_value_data_file_writer_factory.h" +#include "paimon/core/io/shredding_key_value_data_file_writer_factory.h" + +namespace paimon { + +Result> +KeyValueDataFileWriterFactories::Create(const CoreOptions& options, int64_t schema_id, + const std::shared_ptr& write_schema, + int32_t level, FileSource file_source, + const std::vector& primary_keys, + const std::shared_ptr& path_factory, + bool create_stats_extractor, bool is_changelog, + const std::shared_ptr& pool) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan_factory, + ShreddingWritePlanFactories::SelectActive(options, write_schema, pool)); + std::shared_ptr writer_factory; + if (plan_factory != nullptr) { + writer_factory = std::make_shared( + options, schema_id, write_schema, level, file_source, primary_keys, path_factory, + create_stats_extractor, plan_factory, is_changelog, pool); + } else { + writer_factory = std::make_shared( + options, schema_id, write_schema, level, file_source, primary_keys, path_factory, + create_stats_extractor, is_changelog, pool); + } + return writer_factory; +} + +} // namespace paimon diff --git a/src/paimon/core/io/key_value_data_file_writer_factories.h b/src/paimon/core/io/key_value_data_file_writer_factories.h new file mode 100644 index 000000000..335bb94d4 --- /dev/null +++ b/src/paimon/core/io/key_value_data_file_writer_factories.h @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/io/single_file_writer_factory.h" +#include "paimon/core/key_value.h" +#include "paimon/core/manifest/file_source.h" +#include "paimon/result.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +class CoreOptions; +class DataFilePathFactory; +class MemoryPool; + +/// Creates the appropriate key-value data file writer factory for the configured write schema. +class KeyValueDataFileWriterFactories { + public: + using WriterFactory = SingleFileWriterFactory>; + + static Result> Create( + const CoreOptions& options, int64_t schema_id, + const std::shared_ptr& write_schema, int32_t level, FileSource file_source, + const std::vector& primary_keys, + const std::shared_ptr& path_factory, bool create_stats_extractor, + bool is_changelog, const std::shared_ptr& pool); +}; + +} // namespace paimon diff --git a/src/paimon/core/io/key_value_data_file_writer_factory.cpp b/src/paimon/core/io/key_value_data_file_writer_factory.cpp index 8f3885591..e0640ab61 100644 --- a/src/paimon/core/io/key_value_data_file_writer_factory.cpp +++ b/src/paimon/core/io/key_value_data_file_writer_factory.cpp @@ -20,6 +20,7 @@ #include "paimon/core/io/key_value_data_file_writer_factory.h" #include +#include #include #include "arrow/c/helpers.h" @@ -37,14 +38,15 @@ KeyValueDataFileWriterFactory::KeyValueDataFileWriterFactory( const std::shared_ptr& write_schema, int32_t level, FileSource file_source, const std::vector& primary_keys, const std::shared_ptr& path_factory, bool create_stats_extractor, - const std::shared_ptr& pool) + bool is_changelog, const std::shared_ptr& pool) : DataFileWriterFactory(options, schema_id, pool), write_schema_(write_schema), level_(level), file_source_(file_source), primary_keys_(primary_keys), path_factory_(path_factory), - create_stats_extractor_(create_stats_extractor) {} + create_stats_extractor_(create_stats_extractor), + is_changelog_(is_changelog) {} Result>>> KeyValueDataFileWriterFactory::CreateWriter() const { @@ -54,22 +56,50 @@ KeyValueDataFileWriterFactory::CreateWriter() const { return Status::OK(); }; - auto format = options_.GetWriteFileFormat(level_); + std::shared_ptr format = GetFileFormat(); PAIMON_ASSIGN_OR_RAISE(WriterResources resources, CreateWriterResources(*format, write_schema_, create_stats_extractor_)); auto writer = std::make_unique( - options_.GetWriteFileCompression(level_), std::move(converter), schema_id_, level_, - file_source_, primary_keys_, resources.stats_extractor, write_schema_, - path_factory_->IsExternalPath(), pool_); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_index_writer, - CreateFileIndexWriter(write_schema_, path_factory_)); - if (file_index_writer) { - writer->SetFileIndexWriter(std::move(file_index_writer), write_schema_); + GetFileCompression(), std::move(converter), schema_id_, level_, file_source_, primary_keys_, + resources.stats_extractor, write_schema_, path_factory_->IsExternalPath(), pool_); + if (!is_changelog_) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_index_writer, + CreateFileIndexWriter(write_schema_, path_factory_)); + if (file_index_writer) { + writer->SetFileIndexWriter(std::move(file_index_writer), write_schema_); + } } - PAIMON_RETURN_NOT_OK( - writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), resources.writer_builder)); + PAIMON_RETURN_NOT_OK(writer->Init(options_.GetFileSystem(), NewFilePath(format->Identifier()), + resources.writer_builder)); return std::unique_ptr>>( std::move(writer)); } +std::shared_ptr KeyValueDataFileWriterFactory::GetFileFormat() const { + if (is_changelog_) { + std::shared_ptr changelog_format = options_.GetChangelogFileFormat(); + if (changelog_format) { + return changelog_format; + } + } + return options_.GetWriteFileFormat(level_); +} + +std::string KeyValueDataFileWriterFactory::GetFileCompression() const { + if (is_changelog_) { + std::optional changelog_compression = options_.GetChangelogFileCompression(); + if (changelog_compression) { + return changelog_compression.value(); + } + } + return options_.GetWriteFileCompression(level_); +} + +std::string KeyValueDataFileWriterFactory::NewFilePath(const std::string& format_identifier) const { + if (is_changelog_) { + return path_factory_->NewChangelogPath(options_.ChangelogFilePrefix(), format_identifier); + } + return path_factory_->NewPath(); +} + } // namespace paimon diff --git a/src/paimon/core/io/key_value_data_file_writer_factory.h b/src/paimon/core/io/key_value_data_file_writer_factory.h index 6ac50aba3..533b84c6e 100644 --- a/src/paimon/core/io/key_value_data_file_writer_factory.h +++ b/src/paimon/core/io/key_value_data_file_writer_factory.h @@ -38,6 +38,7 @@ namespace paimon { class CoreOptions; class DataFilePathFactory; +class FileFormat; class MemoryPool; class KeyValueDataFileWriterFactory @@ -49,19 +50,24 @@ class KeyValueDataFileWriterFactory FileSource file_source, const std::vector& primary_keys, const std::shared_ptr& path_factory, - bool create_stats_extractor, + bool create_stats_extractor, bool is_changelog, const std::shared_ptr& pool); Result>>> CreateWriter() const override; protected: + std::shared_ptr GetFileFormat() const; + std::string GetFileCompression() const; + std::string NewFilePath(const std::string& format_identifier) const; + std::shared_ptr write_schema_; int32_t level_; FileSource file_source_; std::vector primary_keys_; std::shared_ptr path_factory_; bool create_stats_extractor_; + bool is_changelog_; }; } // namespace paimon diff --git a/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp b/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp index 30d4c9fce..f56ba8d14 100644 --- a/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp +++ b/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp @@ -37,10 +37,11 @@ ShreddingKeyValueDataFileWriterFactory::ShreddingKeyValueDataFileWriterFactory( const std::shared_ptr& write_schema, int32_t level, FileSource file_source, const std::vector& primary_keys, const std::shared_ptr& path_factory, bool create_stats_extractor, - const std::shared_ptr& plan_factory, + const std::shared_ptr& plan_factory, bool is_changelog, const std::shared_ptr& pool) : KeyValueDataFileWriterFactory(options, schema_id, write_schema, level, file_source, - primary_keys, path_factory, create_stats_extractor, pool), + primary_keys, path_factory, create_stats_extractor, + is_changelog, pool), plan_factory_(plan_factory) {} Result>>> @@ -48,7 +49,7 @@ ShreddingKeyValueDataFileWriterFactory::CreateWriter() const { if (!plan_factory_) { return Status::Invalid("Shredding key-value writer requires a write-plan factory."); } - const std::string format_identifier = options_.GetWriteFileFormat(level_)->Identifier(); + const std::string format_identifier = GetFileFormat()->Identifier(); if (plan_factory_->ShouldInferWritePlan()) { auto create_inner = [this](const std::shared_ptr& converter) { return CreateShreddedWriter(converter); @@ -74,7 +75,8 @@ ShreddingKeyValueDataFileWriterFactory::CreateShreddedWriter( [factory = plan_factory_, converter]() { return factory->OnFileCompleted(converter); }); return writer; } - auto format = options_.GetWriteFileFormat(level_); + std::shared_ptr format = GetFileFormat(); + std::string compression = GetFileCompression(); std::shared_ptr file_schema = converter->GetPhysicalSchema(); std::function batch_converter = [converter](KeyValueBatch key_value_batch, ::ArrowArray* array) -> Status { @@ -86,18 +88,19 @@ ShreddingKeyValueDataFileWriterFactory::CreateShreddedWriter( PAIMON_ASSIGN_OR_RAISE(WriterResources resources, CreateWriterResources(*format, file_schema, create_stats_extractor_)); auto writer = std::make_unique( - options_.GetWriteFileCompression(level_), std::move(batch_converter), schema_id_, level_, - file_source_, primary_keys_, resources.stats_extractor, file_schema, - path_factory_->IsExternalPath(), pool_); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_index_writer, - CreateFileIndexWriter(write_schema_, path_factory_)); - if (file_index_writer) { - writer->SetFileIndexWriter(std::move(file_index_writer), write_schema_); + compression, std::move(batch_converter), schema_id_, level_, file_source_, primary_keys_, + resources.stats_extractor, file_schema, path_factory_->IsExternalPath(), pool_); + if (!is_changelog_) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_index_writer, + CreateFileIndexWriter(write_schema_, path_factory_)); + if (file_index_writer) { + writer->SetFileIndexWriter(std::move(file_index_writer), write_schema_); + } } - PAIMON_RETURN_NOT_OK( - writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), resources.writer_builder)); + PAIMON_RETURN_NOT_OK(writer->Init(options_.GetFileSystem(), NewFilePath(format->Identifier()), + resources.writer_builder)); ShreddingWritePlanFactory::MetadataFinalizer finalizer = - plan_factory_->CreateMetadataFinalizer(converter, options_.GetWriteFileCompression(level_)); + plan_factory_->CreateMetadataFinalizer(converter, compression); if (finalizer) { writer->SetMetadataFinalizer(std::move(finalizer)); } diff --git a/src/paimon/core/io/shredding_key_value_data_file_writer_factory.h b/src/paimon/core/io/shredding_key_value_data_file_writer_factory.h index fb967588e..e32dc8a04 100644 --- a/src/paimon/core/io/shredding_key_value_data_file_writer_factory.h +++ b/src/paimon/core/io/shredding_key_value_data_file_writer_factory.h @@ -41,7 +41,7 @@ class ShreddingKeyValueDataFileWriterFactory : public KeyValueDataFileWriterFact const std::shared_ptr& write_schema, int32_t level, FileSource file_source, const std::vector& primary_keys, const std::shared_ptr& path_factory, bool create_stats_extractor, - const std::shared_ptr& plan_factory, + const std::shared_ptr& plan_factory, bool is_changelog, const std::shared_ptr& pool); Result>>> diff --git a/src/paimon/core/manifest/manifest_list.h b/src/paimon/core/manifest/manifest_list.h index 31959a5c9..ff67e8080 100644 --- a/src/paimon/core/manifest/manifest_list.h +++ b/src/paimon/core/manifest/manifest_list.h @@ -113,8 +113,7 @@ class ManifestList : public ObjectsFile { const std::optional& changelog_manifest_list = snapshot.ChangelogManifestList(); if (changelog_manifest_list) { - return Status::NotImplemented("do not support read changelog manifest list"); - // return Read(changelog_manifest_list.value(), /*filter=*/nullptr, manifests); + return Read(changelog_manifest_list.value(), /*filter=*/nullptr, manifests); } else { return Status::OK(); } diff --git a/src/paimon/core/manifest/manifest_list_test.cpp b/src/paimon/core/manifest/manifest_list_test.cpp index 889f18f2b..23949bfdd 100644 --- a/src/paimon/core/manifest/manifest_list_test.cpp +++ b/src/paimon/core/manifest/manifest_list_test.cpp @@ -25,6 +25,7 @@ #include "gtest/gtest.h" #include "paimon/core/core_options.h" #include "paimon/core/manifest/manifest_file_meta.h" +#include "paimon/core/snapshot.h" #include "paimon/core/stats/simple_stats.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/format/file_format.h" @@ -141,6 +142,33 @@ TEST_F(ManifestListTest, TestEmptyManifestList) { ASSERT_EQ(manifest_file_metas.size(), 0); } +TEST_F(ManifestListTest, TestReadChangelogManifests) { + auto pool = GetDefaultPool(); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto manifest_list = CreateManifestList("orc", dir->Str(), pool); + ManifestFileMeta expected_meta( + "changelog-manifest", /*file_size=*/100, /*num_added_files=*/1, + /*num_deleted_files=*/0, SimpleStats::EmptyStats(), /*schema_id=*/0, + /*min_bucket=*/0, /*max_bucket=*/0, /*min_level=*/0, /*max_level=*/0, + /*min_row_id=*/std::nullopt, /*max_row_id=*/std::nullopt); + ASSERT_OK_AND_ASSIGN(auto changelog_manifest_list, manifest_list->Write({expected_meta})); + Snapshot snapshot( + /*id=*/1, /*schema_id=*/0, /*base_manifest_list=*/"", + /*base_manifest_list_size=*/std::nullopt, /*delta_manifest_list=*/"", + /*delta_manifest_list_size=*/std::nullopt, + /*changelog_manifest_list=*/changelog_manifest_list.first, + /*changelog_manifest_list_size=*/changelog_manifest_list.second, + /*index_manifest=*/std::nullopt, /*commit_user=*/"user", /*commit_identifier=*/1, + Snapshot::CommitKind::Append(), /*time_millis=*/0, /*total_record_count=*/1, + /*delta_record_count=*/1, /*changelog_record_count=*/1, /*watermark=*/std::nullopt, + /*statistics=*/std::nullopt, /*properties=*/std::nullopt, /*next_row_id=*/std::nullopt); + + std::vector actual_metas; + ASSERT_OK(manifest_list->ReadChangelogManifests(snapshot, &actual_metas)); + ASSERT_EQ(std::vector({expected_meta}), actual_metas); +} + TEST_F(ManifestListTest, TestManifestListCompatibleWithJavaPaimon09) { auto pool = GetDefaultPool(); auto manifest_file_metas = ReadManifestFileMeta("avro", paimon::test::GetDataDir() + "/avro", diff --git a/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp b/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp index 3efd40f64..4f632425b 100644 --- a/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp +++ b/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp @@ -17,7 +17,142 @@ */ #include "paimon/core/mergetree/compact/changelog_merge_tree_rewriter.h" + +#include + +#include "paimon/common/utils/fields_comparator.h" +#include "paimon/common/utils/scope_guard.h" +#include "paimon/core/io/key_value_meta_projection_consumer.h" +#include "paimon/core/io/row_to_arrow_array_converter.h" +#include "paimon/format/file_format.h" namespace paimon { + +namespace { + +using CancellationChecker = std::function; + +class ChangelogCompactionBatchProducer : public AsyncKeyValueBatchProducer { + public: + ChangelogCompactionBatchProducer( + std::unique_ptr&& sort_merge_reader, int32_t write_batch_size, + std::shared_ptr>&& merge_function_wrapper, + const FieldsComparator::FieldComparatorFunc& key_comparator, + const CancellationChecker& cancellation_checker, bool drop_delete, bool produce_data, + bool produce_changelog) + : sort_merge_reader_(std::move(sort_merge_reader)), + write_batch_size_(NormalizeProjectionBatchSize(write_batch_size)), + merge_function_wrapper_(std::move(merge_function_wrapper)), + key_comparator_(key_comparator), + cancellation_checker_(cancellation_checker), + drop_delete_(drop_delete), + produce_data_(produce_data), + produce_changelog_(produce_changelog) {} + + Status Produce(AsyncKeyValueBatchSink* sink) override { + std::vector compact_buffer; + std::vector changelog_buffer; + compact_buffer.reserve(write_batch_size_); + changelog_buffer.reserve(write_batch_size_); + + auto flush = [&](AsyncKeyValueBatchType type, std::vector* buffer) -> Status { + if (buffer->empty()) { + return Status::OK(); + } + std::vector rows = std::move(*buffer); + buffer->clear(); + buffer->reserve(write_batch_size_); + return sink->Write(type, std::move(rows)); + }; + + auto emit_result = [&](ChangelogResult&& result) -> Status { + if (produce_data_ && result.result && + (!drop_delete_ || result.result->value_kind->IsAdd())) { + compact_buffer.emplace_back(std::move(result.result).value()); + if (static_cast(compact_buffer.size()) >= write_batch_size_) { + PAIMON_RETURN_NOT_OK(flush(AsyncKeyValueBatchType::DATA, &compact_buffer)); + } + } + if (produce_changelog_) { + for (auto& changelog : result.changelogs) { + changelog_buffer.emplace_back(std::move(changelog)); + if (static_cast(changelog_buffer.size()) >= write_batch_size_) { + PAIMON_RETURN_NOT_OK( + flush(AsyncKeyValueBatchType::CHANGELOG, &changelog_buffer)); + } + } + } + return Status::OK(); + }; + + std::shared_ptr current_key; + auto finish_group = [&]() -> Status { + if (!current_key) { + return Status::OK(); + } + PAIMON_ASSIGN_OR_RAISE(std::optional result, + merge_function_wrapper_->GetResult()); + current_key.reset(); + if (result) { + PAIMON_RETURN_NOT_OK(emit_result(std::move(result).value())); + } + return Status::OK(); + }; + + while (true) { + if (cancellation_checker_()) { + return Status::Cancelled("Compaction is cancelled"); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr iterator, + sort_merge_reader_->NextBatch()); + if (!iterator) { + break; + } + while (true) { + PAIMON_ASSIGN_OR_RAISE(bool has_next, iterator->HasNext()); + if (!has_next) { + break; + } + KeyValue key_value = iterator->Next(); + if (current_key && key_comparator_(*current_key, *key_value.key) != 0) { + PAIMON_RETURN_NOT_OK(finish_group()); + } + if (!current_key) { + merge_function_wrapper_->Reset(); + current_key = key_value.key; + } + PAIMON_RETURN_NOT_OK(merge_function_wrapper_->Add(std::move(key_value))); + } + } + PAIMON_RETURN_NOT_OK(finish_group()); + sort_merge_reader_->Close(); + closed_ = true; + PAIMON_RETURN_NOT_OK(flush(AsyncKeyValueBatchType::DATA, &compact_buffer)); + PAIMON_RETURN_NOT_OK(flush(AsyncKeyValueBatchType::CHANGELOG, &changelog_buffer)); + return Status::OK(); + } + + void Close() override { + if (closed_) { + return; + } + sort_merge_reader_->Close(); + closed_ = true; + } + + private: + std::unique_ptr sort_merge_reader_; + int32_t write_batch_size_; + std::shared_ptr> merge_function_wrapper_; + FieldsComparator::FieldComparatorFunc key_comparator_; + CancellationChecker cancellation_checker_; + bool drop_delete_; + bool produce_data_; + bool produce_changelog_; + bool closed_ = false; +}; + +} // namespace + ChangelogMergeTreeRewriter::ChangelogMergeTreeRewriter( int32_t max_level, bool force_drop_delete, const BinaryRow& partition, int32_t bucket, int64_t schema_id, const std::vector& trimmed_primary_keys, @@ -26,14 +161,18 @@ ChangelogMergeTreeRewriter::ChangelogMergeTreeRewriter( const std::shared_ptr& path_factory_cache, std::unique_ptr&& merge_file_split_read, MergeFunctionWrapperFactory merge_function_wrapper_factory, - const std::shared_ptr& cancellation_controller, + ChangelogMergeFunctionWrapperFactory changelog_merge_function_wrapper_factory, + bool produce_changelog, const std::shared_ptr& cancellation_controller, const std::shared_ptr& pool) : MergeTreeCompactRewriter( partition, bucket, schema_id, trimmed_primary_keys, options, data_schema, write_schema, std::move(dv_factory), path_factory_cache, std::move(merge_file_split_read), std::move(merge_function_wrapper_factory), cancellation_controller, pool), max_level_(max_level), - force_drop_delete_(force_drop_delete) {} + force_drop_delete_(force_drop_delete), + changelog_merge_function_wrapper_factory_( + std::move(changelog_merge_function_wrapper_factory)), + produce_changelog_(produce_changelog) {} Result ChangelogMergeTreeRewriter::Rewrite( int32_t output_level, bool drop_delete, const std::vector>& sections) { @@ -78,31 +217,78 @@ Result ChangelogMergeTreeRewriter::RewriteOrProduceChangelog( bool rewrite_compact_file) { PAIMON_ASSIGN_OR_RAISE(MergeTreeCompactRewriter::KeyValueConsumerCreator create_consumer, GenerateKeyValueConsumer()); - std::vector> reader_holders; - auto before = ExtractFilesFromSections(sections); std::unique_ptr compact_file_writer; if (rewrite_compact_file) { PAIMON_ASSIGN_OR_RAISE(compact_file_writer, CreateRollingRowWriter(output_level)); } - // TODO(xinyu.lxy): produce changelog + std::unique_ptr changelog_file_writer; + if (produce_changelog_) { + PAIMON_ASSIGN_OR_RAISE(changelog_file_writer, CreateRollingChangelogWriter(output_level)); + } + + std::vector> reader_holders; ScopeGuard write_guard([&]() -> void { if (compact_file_writer) { compact_file_writer->Abort(); + compact_file_writer.reset(); + } + if (changelog_file_writer) { + changelog_file_writer->Abort(); + changelog_file_writer.reset(); } - merge_file_split_read_.reset(); for (const auto& reader : reader_holders) { reader->Close(); } + merge_file_split_read_.reset(); }); + bool produce_data = compact_file_writer != nullptr; + bool produce_changelog = changelog_file_writer != nullptr; + FieldsComparator::FieldComparatorFunc key_comparator = [this](const InternalRow& lhs, + const InternalRow& rhs) { + return merge_file_split_read_->GetKeyComparator()->CompareTo(lhs, rhs); + }; + CancellationChecker cancellation_checker = [this]() { return IsCancelled(); }; + for (const auto& section : sections) { - PAIMON_RETURN_NOT_OK(MergeReadAndWrite(output_level, drop_delete, section, create_consumer, - compact_file_writer.get(), &reader_holders)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr sort_merge_reader, + CreateRawSortMergeReaderForSection(section)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr> merge_function_wrapper, + changelog_merge_function_wrapper_factory_(output_level)); + std::unique_ptr producer = + std::make_unique( + std::move(sort_merge_reader), options_.GetWriteBatchSize(), + std::move(merge_function_wrapper), key_comparator, cancellation_checker, + drop_delete, produce_data, produce_changelog); + auto producer_and_consumer = + std::make_shared>( + std::move(producer), create_consumer, /*consumer_thread_num=*/1); + reader_holders.emplace_back(producer_and_consumer); + + while (true) { + if (IsCancelled()) { + return Status::Cancelled("Compaction is cancelled"); + } + PAIMON_ASSIGN_OR_RAISE(AsyncKeyValueResultBatch output, + producer_and_consumer->NextBatchWithType()); + if (output.result.batch == nullptr) { + break; + } + if (output.type == AsyncKeyValueBatchType::DATA) { + PAIMON_RETURN_NOT_OK(compact_file_writer->Write(std::move(output.result))); + } else { + PAIMON_RETURN_NOT_OK(changelog_file_writer->Write(std::move(output.result))); + } + } } if (compact_file_writer) { PAIMON_RETURN_NOT_OK(compact_file_writer->Close()); } + if (changelog_file_writer) { + PAIMON_RETURN_NOT_OK(changelog_file_writer->Close()); + } std::vector> after; if (compact_file_writer) { PAIMON_ASSIGN_OR_RAISE(after, compact_file_writer->GetResult()); @@ -118,8 +304,12 @@ Result ChangelogMergeTreeRewriter::RewriteOrProduceChangelog( NotifyRewriteCompactBefore(before); } PAIMON_ASSIGN_OR_RAISE(after, NotifyRewriteCompactAfter(after)); + std::vector> changelog_files; + if (changelog_file_writer) { + PAIMON_ASSIGN_OR_RAISE(changelog_files, changelog_file_writer->GetResult()); + } write_guard.Release(); - return CompactResult(before, after); + return CompactResult(before, after, changelog_files); } } // namespace paimon diff --git a/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.h b/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.h index c5d8e8914..c080b30a7 100644 --- a/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.h +++ b/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.h @@ -22,11 +22,15 @@ #include "paimon/core/core_options.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/core/key_value.h" +#include "paimon/core/mergetree/compact/changelog_result.h" #include "paimon/core/mergetree/compact/merge_tree_compact_rewriter.h" namespace paimon { /// A `MergeTreeCompactRewriter` which produces changelog files while performing compaction. class ChangelogMergeTreeRewriter : public MergeTreeCompactRewriter { public: + using ChangelogMergeFunctionWrapperFactory = + std::function>>(int32_t)>; + Result Rewrite(int32_t output_level, bool drop_delete, const std::vector>& sections) override; @@ -42,6 +46,8 @@ class ChangelogMergeTreeRewriter : public MergeTreeCompactRewriter { const std::shared_ptr& path_factory_cache, std::unique_ptr&& merge_file_split_read, MergeFunctionWrapperFactory merge_function_wrapper_factory, + ChangelogMergeFunctionWrapperFactory changelog_merge_function_wrapper_factory, + bool produce_changelog, const std::shared_ptr& cancellation_controller, const std::shared_ptr& pool); @@ -85,5 +91,8 @@ class ChangelogMergeTreeRewriter : public MergeTreeCompactRewriter { Result RewriteOrProduceChangelog( int32_t output_level, const std::vector>& sections, bool drop_delete, bool rewrite_compact_file); + + ChangelogMergeFunctionWrapperFactory changelog_merge_function_wrapper_factory_; + bool produce_changelog_; }; } // namespace paimon diff --git a/src/paimon/core/mergetree/compact/changelog_result.h b/src/paimon/core/mergetree/compact/changelog_result.h new file mode 100644 index 000000000..778183ed7 --- /dev/null +++ b/src/paimon/core/mergetree/compact/changelog_result.h @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/core/key_value.h" + +namespace paimon { + +/// The result of merging all records with the same primary key while producing changelog. +struct ChangelogResult { + std::optional result; + std::vector changelogs; +}; + +} // namespace paimon diff --git a/src/paimon/core/mergetree/compact/first_row_merge_function_wrapper.h b/src/paimon/core/mergetree/compact/first_row_merge_function_wrapper.h index 319ed5ab6..e0d7e6d7f 100644 --- a/src/paimon/core/mergetree/compact/first_row_merge_function_wrapper.h +++ b/src/paimon/core/mergetree/compact/first_row_merge_function_wrapper.h @@ -24,7 +24,9 @@ #include #include +#include "paimon/common/data/serializer/row_compacted_serializer.h" #include "paimon/core/key_value.h" +#include "paimon/core/mergetree/compact/changelog_result.h" #include "paimon/core/mergetree/compact/first_row_merge_function.h" #include "paimon/core/mergetree/compact/merge_function_wrapper.h" #include "paimon/result.h" @@ -32,12 +34,15 @@ namespace paimon { /// Wrapper for `MergeFunction`s to produce changelog by lookup for first row. -class FirstRowMergeFunctionWrapper : public MergeFunctionWrapper { +class FirstRowMergeFunctionWrapper : public MergeFunctionWrapper { public: FirstRowMergeFunctionWrapper( std::unique_ptr&& merge_function, - std::function(const std::shared_ptr&)> contains) - : merge_function_(std::move(merge_function)), contains_(std::move(contains)) {} + std::function(const std::shared_ptr&)> contains, + std::unique_ptr&& value_serializer) + : merge_function_(std::move(merge_function)), + contains_(std::move(contains)), + value_serializer_(std::move(value_serializer)) {} void Reset() override { merge_function_->Reset(); @@ -47,11 +52,13 @@ class FirstRowMergeFunctionWrapper : public MergeFunctionWrapper { return merge_function_->Add(std::move(kv)); } - Result> GetResult() override { + Result> GetResult() override { PAIMON_ASSIGN_OR_RAISE(std::optional result, merge_function_->GetResult()); + ChangelogResult changelog_result; if (merge_function_->ContainsHighLevel()) { + changelog_result.result = std::move(result); Reset(); - return result; + return std::optional(std::move(changelog_result)); } if (!result) { Reset(); @@ -62,17 +69,27 @@ class FirstRowMergeFunctionWrapper : public MergeFunctionWrapper { if (contains) { // empty Reset(); - return std::optional(); + return std::optional(std::move(changelog_result)); } - // new record, output changelog - // TODO(xinyu.lxy) support changelog + // TODO(lisizhuo.lsz): avoid serialize & deserialize here. + if (value_serializer_) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr bytes, + value_serializer_->SerializeToBytes(*result->value)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr changelog_value, + value_serializer_->Deserialize(bytes)); + changelog_result.changelogs.emplace_back(result->value_kind, result->sequence_number, + result->level, result->key, + std::move(changelog_value)); + } + changelog_result.result = std::move(result); Reset(); - return result; + return std::optional(std::move(changelog_result)); } private: std::unique_ptr merge_function_; std::function(const std::shared_ptr&)> contains_; + std::unique_ptr value_serializer_; }; } // namespace paimon diff --git a/src/paimon/core/mergetree/compact/first_row_merge_function_wrapper_test.cpp b/src/paimon/core/mergetree/compact/first_row_merge_function_wrapper_test.cpp index 36a2ef01e..47a2d38ed 100644 --- a/src/paimon/core/mergetree/compact/first_row_merge_function_wrapper_test.cpp +++ b/src/paimon/core/mergetree/compact/first_row_merge_function_wrapper_test.cpp @@ -29,6 +29,15 @@ #include "paimon/testing/utils/testharness.h" namespace paimon::test { +namespace { +std::unique_ptr CreateValueSerializer( + const std::shared_ptr& pool) { + return RowCompactedSerializer::Create(arrow::schema({arrow::field("value", arrow::int32())}), + pool) + .value(); +} +} // namespace + TEST(FirstRowMergeFunctionWrapperTest, TestSimple) { auto pool = GetDefaultPool(); KeyValue kv1(RowKind::Insert(), /*sequence_number=*/0, /*level=*/0, /*key=*/ @@ -45,14 +54,17 @@ TEST(FirstRowMergeFunctionWrapperTest, TestSimple) { auto contains = [](const std::shared_ptr& row) { return true; }; - FirstRowMergeFunctionWrapper wrapper(std::move(mfunc), std::move(contains)); + FirstRowMergeFunctionWrapper wrapper(std::move(mfunc), std::move(contains), + CreateValueSerializer(pool)); wrapper.Reset(); ASSERT_OK(wrapper.Add(std::move(kv1))); ASSERT_OK(wrapper.Add(std::move(kv2))); ASSERT_OK(wrapper.Add(std::move(kv3))); ASSERT_OK_AND_ASSIGN(auto result, wrapper.GetResult()); ASSERT_TRUE(result); - ASSERT_EQ(result.value().sequence_number, 0); + ASSERT_TRUE(result->result); + ASSERT_EQ(result->result->sequence_number, 0); + ASSERT_TRUE(result->changelogs.empty()); } TEST(FirstRowMergeFunctionWrapperTest, TestAllLevel0WithContain) { @@ -71,13 +83,16 @@ TEST(FirstRowMergeFunctionWrapperTest, TestAllLevel0WithContain) { auto contains = [](const std::shared_ptr& row) { return true; }; - FirstRowMergeFunctionWrapper wrapper(std::move(mfunc), std::move(contains)); + FirstRowMergeFunctionWrapper wrapper(std::move(mfunc), std::move(contains), + CreateValueSerializer(pool)); wrapper.Reset(); ASSERT_OK(wrapper.Add(std::move(kv1))); ASSERT_OK(wrapper.Add(std::move(kv2))); ASSERT_OK(wrapper.Add(std::move(kv3))); ASSERT_OK_AND_ASSIGN(auto result, wrapper.GetResult()); - ASSERT_FALSE(result); + ASSERT_TRUE(result); + ASSERT_FALSE(result->result); + ASSERT_TRUE(result->changelogs.empty()); } TEST(FirstRowMergeFunctionWrapperTest, TestAllLevel0WithoutContain) { @@ -96,14 +111,18 @@ TEST(FirstRowMergeFunctionWrapperTest, TestAllLevel0WithoutContain) { auto contains = [](const std::shared_ptr& row) { return false; }; - FirstRowMergeFunctionWrapper wrapper(std::move(mfunc), std::move(contains)); + FirstRowMergeFunctionWrapper wrapper(std::move(mfunc), std::move(contains), + CreateValueSerializer(pool)); wrapper.Reset(); ASSERT_OK(wrapper.Add(std::move(kv1))); ASSERT_OK(wrapper.Add(std::move(kv2))); ASSERT_OK(wrapper.Add(std::move(kv3))); ASSERT_OK_AND_ASSIGN(auto result, wrapper.GetResult()); ASSERT_TRUE(result); - ASSERT_EQ(result.value().sequence_number, 0); + ASSERT_TRUE(result->result); + ASSERT_EQ(result->result->sequence_number, 0); + ASSERT_EQ(result->changelogs.size(), 1); + ASSERT_EQ(result->changelogs[0].sequence_number, 0); } } // namespace paimon::test diff --git a/src/paimon/core/mergetree/compact/internal_row_equalizer.h b/src/paimon/core/mergetree/compact/internal_row_equalizer.h new file mode 100644 index 000000000..22f15d8bd --- /dev/null +++ b/src/paimon/core/mergetree/compact/internal_row_equalizer.h @@ -0,0 +1,250 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "fmt/format.h" +#include "paimon/common/data/data_getters.h" +#include "paimon/common/data/internal_array.h" +#include "paimon/common/data/internal_map.h" +#include "paimon/common/data/internal_row.h" +#include "paimon/common/utils/checked_cast.h" +#include "paimon/common/utils/date_time_utils.h" +#include "paimon/common/utils/fields_comparator.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +/// Creates equality functions for internal rows, including nested values. +/// Java's RecordEqualiser also compares the RowKind embedded in InternalRow. This comparator +/// currently compares field values only. This does not affect the current lookup changelog results +/// because changelog kinds are tracked separately by KeyValue::value_kind. +class InternalRowEqualizer { + public: + static Result Create( + const std::shared_ptr& schema, + const std::vector& ignore_fields) { + std::set ignored(ignore_fields.begin(), ignore_fields.end()); + std::vector> equalizers; + for (int32_t i = 0; i < schema->num_fields(); ++i) { + if (ignored.find(schema->field(i)->name()) != ignored.end()) { + continue; + } + PAIMON_ASSIGN_OR_RAISE(ValueEqualizer equalizer, + CreateValueEqualizer(schema->field(i)->type())); + equalizers.emplace_back(i, std::move(equalizer)); + } + return FieldsComparator::FieldComparatorFunc( + [equalizers = std::move(equalizers)](const InternalRow& lhs, const InternalRow& rhs) { + for (const auto& [field_idx, equalizer] : equalizers) { + if (!EqualAt(lhs, field_idx, rhs, field_idx, equalizer)) { + return 1; + } + } + return 0; + }); + } + + private: + using ValueEqualizer = + std::function; + + static bool EqualAt(const DataGetters& lhs, int32_t lhs_pos, const DataGetters& rhs, + int32_t rhs_pos, const ValueEqualizer& equalizer) { + bool lhs_null = lhs.IsNullAt(lhs_pos); + bool rhs_null = rhs.IsNullAt(rhs_pos); + if (lhs_null || rhs_null) { + return lhs_null == rhs_null; + } + return equalizer(lhs, lhs_pos, rhs, rhs_pos); + } + + static Result CreateValueEqualizer( + const std::shared_ptr& type) { + switch (type->id()) { + case arrow::Type::BOOL: + return PrimitiveEqualizer( + [](const DataGetters& row, int32_t pos) { return row.GetBoolean(pos); }); + case arrow::Type::INT8: + return PrimitiveEqualizer( + [](const DataGetters& row, int32_t pos) { return row.GetByte(pos); }); + case arrow::Type::INT16: + return PrimitiveEqualizer( + [](const DataGetters& row, int32_t pos) { return row.GetShort(pos); }); + case arrow::Type::INT32: + return PrimitiveEqualizer( + [](const DataGetters& row, int32_t pos) { return row.GetInt(pos); }); + case arrow::Type::DATE32: + return PrimitiveEqualizer( + [](const DataGetters& row, int32_t pos) { return row.GetDate(pos); }); + case arrow::Type::INT64: + return PrimitiveEqualizer( + [](const DataGetters& row, int32_t pos) { return row.GetLong(pos); }); + case arrow::Type::FLOAT: + return ValueEqualizer([](const DataGetters& lhs, int32_t lhs_pos, + const DataGetters& rhs, int32_t rhs_pos) { + return FieldsComparator::CompareFloatingPoint(lhs.GetFloat(lhs_pos), + rhs.GetFloat(rhs_pos)) == 0; + }); + case arrow::Type::DOUBLE: + return ValueEqualizer([](const DataGetters& lhs, int32_t lhs_pos, + const DataGetters& rhs, int32_t rhs_pos) { + return FieldsComparator::CompareFloatingPoint(lhs.GetDouble(lhs_pos), + rhs.GetDouble(rhs_pos)) == 0; + }); + case arrow::Type::STRING: + case arrow::Type::BINARY: + return ValueEqualizer([](const DataGetters& lhs, int32_t lhs_pos, + const DataGetters& rhs, int32_t rhs_pos) { + return lhs.GetStringView(lhs_pos) == rhs.GetStringView(rhs_pos); + }); + case arrow::Type::TIMESTAMP: { + std::shared_ptr timestamp_type = + checked_pointer_cast(type); + int32_t precision = DateTimeUtils::GetPrecisionFromType(timestamp_type); + return ValueEqualizer([precision](const DataGetters& lhs, int32_t lhs_pos, + const DataGetters& rhs, int32_t rhs_pos) { + return lhs.GetTimestamp(lhs_pos, precision) == + rhs.GetTimestamp(rhs_pos, precision); + }); + } + case arrow::Type::DECIMAL128: { + std::shared_ptr decimal_type = + checked_pointer_cast(type); + int32_t precision = decimal_type->precision(); + int32_t scale = decimal_type->scale(); + return ValueEqualizer([precision, scale](const DataGetters& lhs, int32_t lhs_pos, + const DataGetters& rhs, int32_t rhs_pos) { + return lhs.GetDecimal(lhs_pos, precision, scale) + .CompareTo(rhs.GetDecimal(rhs_pos, precision, scale)) == 0; + }); + } + case arrow::Type::LIST: { + std::shared_ptr list_type = + checked_pointer_cast(type); + PAIMON_ASSIGN_OR_RAISE(ValueEqualizer element_equalizer, + CreateValueEqualizer(list_type->value_type())); + return ValueEqualizer([element_equalizer = std::move(element_equalizer)]( + const DataGetters& lhs, int32_t lhs_pos, + const DataGetters& rhs, int32_t rhs_pos) { + std::shared_ptr lhs_array = lhs.GetArray(lhs_pos); + std::shared_ptr rhs_array = rhs.GetArray(rhs_pos); + if (lhs_array->Size() != rhs_array->Size()) { + return false; + } + for (int32_t i = 0; i < lhs_array->Size(); ++i) { + if (!EqualAt(*lhs_array, i, *rhs_array, i, element_equalizer)) { + return false; + } + } + return true; + }); + } + case arrow::Type::MAP: { + std::shared_ptr map_type = + checked_pointer_cast(type); + PAIMON_ASSIGN_OR_RAISE(ValueEqualizer key_equalizer, + CreateValueEqualizer(map_type->key_type())); + PAIMON_ASSIGN_OR_RAISE(ValueEqualizer item_equalizer, + CreateValueEqualizer(map_type->item_type())); + return ValueEqualizer([key_equalizer = std::move(key_equalizer), + item_equalizer = std::move(item_equalizer)]( + const DataGetters& lhs, int32_t lhs_pos, + const DataGetters& rhs, int32_t rhs_pos) { + std::shared_ptr lhs_map = lhs.GetMap(lhs_pos); + std::shared_ptr rhs_map = rhs.GetMap(rhs_pos); + if (lhs_map->Size() != rhs_map->Size()) { + return false; + } + std::shared_ptr lhs_keys = lhs_map->KeyArray(); + std::shared_ptr rhs_keys = rhs_map->KeyArray(); + std::shared_ptr lhs_values = lhs_map->ValueArray(); + std::shared_ptr rhs_values = rhs_map->ValueArray(); + std::vector matched(rhs_map->Size(), false); + for (int32_t lhs_index = 0; lhs_index < lhs_map->Size(); ++lhs_index) { + bool found = false; + for (int32_t rhs_index = 0; rhs_index < rhs_map->Size(); ++rhs_index) { + if (matched[rhs_index]) { + continue; + } + if (EqualAt(*lhs_keys, lhs_index, *rhs_keys, rhs_index, + key_equalizer) && + EqualAt(*lhs_values, lhs_index, *rhs_values, rhs_index, + item_equalizer)) { + matched[rhs_index] = true; + found = true; + break; + } + } + if (!found) { + return false; + } + } + return true; + }); + } + case arrow::Type::STRUCT: { + std::shared_ptr struct_type = + checked_pointer_cast(type); + std::vector field_equalizers; + field_equalizers.reserve(struct_type->num_fields()); + for (const auto& field : struct_type->fields()) { + PAIMON_ASSIGN_OR_RAISE(ValueEqualizer field_equalizer, + CreateValueEqualizer(field->type())); + field_equalizers.emplace_back(std::move(field_equalizer)); + } + int32_t field_count = struct_type->num_fields(); + return ValueEqualizer([field_equalizers = std::move(field_equalizers), field_count]( + const DataGetters& lhs, int32_t lhs_pos, + const DataGetters& rhs, int32_t rhs_pos) { + std::shared_ptr lhs_row = lhs.GetRow(lhs_pos, field_count); + std::shared_ptr rhs_row = rhs.GetRow(rhs_pos, field_count); + for (int32_t i = 0; i < field_count; ++i) { + if (!EqualAt(*lhs_row, i, *rhs_row, i, field_equalizers[i])) { + return false; + } + } + return true; + }); + } + default: + return Status::NotImplemented( + fmt::format("Do not support equality for type {}", type->ToString())); + } + } + + template + static ValueEqualizer PrimitiveEqualizer(Getter getter) { + return [getter = std::move(getter)](const DataGetters& lhs, int32_t lhs_pos, + const DataGetters& rhs, int32_t rhs_pos) { + return static_cast(getter(lhs, lhs_pos)) == static_cast(getter(rhs, rhs_pos)); + }; + } +}; + +} // namespace paimon diff --git a/src/paimon/core/mergetree/compact/internal_row_equalizer_test.cpp b/src/paimon/core/mergetree/compact/internal_row_equalizer_test.cpp new file mode 100644 index 000000000..80bc37b55 --- /dev/null +++ b/src/paimon/core/mergetree/compact/internal_row_equalizer_test.cpp @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/mergetree/compact/internal_row_equalizer.h" + +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "gtest/gtest.h" +#include "paimon/common/data/binary_array.h" +#include "paimon/common/data/binary_array_writer.h" +#include "paimon/common/data/binary_map.h" +#include "paimon/common/data/generic_row.h" +#include "paimon/common/data/internal_array.h" +#include "paimon/common/data/internal_map.h" +#include "paimon/common/data/internal_row.h" +#include "paimon/common/utils/decimal_utils.h" +#include "paimon/data/decimal.h" +#include "paimon/data/timestamp.h" +#include "paimon/memory/bytes.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { +namespace { + +std::shared_ptr CreateIntArray(const std::vector& values, + MemoryPool* pool) { + return std::make_shared(BinaryArray::FromIntArray(values, pool)); +} + +std::shared_ptr CreateNullableIntArray(const std::vector& values, + int32_t null_pos, MemoryPool* pool) { + auto array = std::make_shared(); + BinaryArrayWriter writer(array.get(), static_cast(values.size()), sizeof(int32_t), + pool); + for (int32_t i = 0; i < static_cast(values.size()); ++i) { + if (i == null_pos) { + writer.SetNullValue(i); + } else { + writer.WriteInt(i, values[i]); + } + } + writer.Complete(); + return array; +} + +std::shared_ptr CreateIntMap(const std::vector& keys, + const std::vector& values, MemoryPool* pool) { + BinaryArray key_array = BinaryArray::FromIntArray(keys, pool); + BinaryArray value_array = BinaryArray::FromIntArray(values, pool); + return BinaryMap::ValueOf(key_array, value_array, pool); +} + +std::shared_ptr CreateNestedRow(int32_t value, double floating_point) { + return GenericRow::Of({value, floating_point}); +} + +} // namespace + +TEST(InternalRowEqualizerTest, PrimitiveTypesAndIgnoreFields) { + std::shared_ptr pool = GetDefaultPool(); + std::shared_ptr schema = arrow::schema( + {arrow::field("boolean", arrow::boolean()), arrow::field("tinyint", arrow::int8()), + arrow::field("smallint", arrow::int16()), arrow::field("int", arrow::int32()), + arrow::field("date", arrow::date32()), arrow::field("bigint", arrow::int64()), + arrow::field("float", arrow::float32()), arrow::field("double", arrow::float64()), + arrow::field("string", arrow::utf8()), arrow::field("binary", arrow::binary()), + arrow::field("timestamp", arrow::timestamp(arrow::TimeUnit::MICRO)), + arrow::field("decimal", arrow::decimal128(10, 2)), + arrow::field("ignored", arrow::int32())}); + + const float float_nan = std::numeric_limits::quiet_NaN(); + const double double_nan = std::numeric_limits::quiet_NaN(); + auto binary = std::make_shared("binary", pool.get()); + Decimal decimal(10, 2, DecimalUtils::StrToInt128("12345").value()); + std::vector left_values = {true, + static_cast(1), + static_cast(2), + static_cast(3), + int32_t{4}, + int64_t{5}, + float_nan, + double_nan, + std::string_view("string"), + binary, + Timestamp(1234, 567000), + decimal, + int32_t{10}}; + std::vector right_values = left_values; + right_values.back() = int32_t{20}; + + std::unique_ptr left = GenericRow::Of(left_values); + std::unique_ptr right = GenericRow::Of(right_values); + ASSERT_OK_AND_ASSIGN(FieldsComparator::FieldComparatorFunc equalizer, + InternalRowEqualizer::Create(schema, {"ignored"})); + ASSERT_EQ(0, equalizer(*left, *right)); + + right->SetField(/*pos=*/3, int32_t{30}); + ASSERT_NE(0, equalizer(*left, *right)); +} + +TEST(InternalRowEqualizerTest, NullAndFloatingPointSemantics) { + std::shared_ptr schema = arrow::schema( + {arrow::field("value", arrow::float64()), arrow::field("nullable", arrow::int32())}); + ASSERT_OK_AND_ASSIGN(FieldsComparator::FieldComparatorFunc equalizer, + InternalRowEqualizer::Create(schema, {})); + + std::unique_ptr negative_zero = + GenericRow::Of({static_cast(-0.0), NullType()}); + std::unique_ptr positive_zero = + GenericRow::Of({static_cast(0.0), NullType()}); + ASSERT_NE(0, equalizer(*negative_zero, *positive_zero)); + + double nan1 = std::numeric_limits::quiet_NaN(); + double nan2 = -std::numeric_limits::quiet_NaN(); + std::unique_ptr left_nan = GenericRow::Of({nan1, NullType()}); + std::unique_ptr right_nan = GenericRow::Of({nan2, NullType()}); + ASSERT_EQ(0, equalizer(*left_nan, *right_nan)); + + right_nan->SetField(/*pos=*/1, int32_t{1}); + ASSERT_NE(0, equalizer(*left_nan, *right_nan)); +} + +TEST(InternalRowEqualizerTest, NestedTypes) { + std::shared_ptr pool = GetDefaultPool(); + std::shared_ptr schema = arrow::schema( + {arrow::field("array", arrow::list(arrow::int32())), + arrow::field("map", arrow::map(arrow::int32(), arrow::int32())), + arrow::field("row", arrow::struct_({arrow::field("value", arrow::int32()), + arrow::field("floating", arrow::float64())}))}); + ASSERT_OK_AND_ASSIGN(FieldsComparator::FieldComparatorFunc equalizer, + InternalRowEqualizer::Create(schema, {})); + + std::unique_ptr left = + GenericRow::Of({CreateNullableIntArray({1, 0, 3}, /*null_pos=*/1, pool.get()), + CreateIntMap({1, 2}, {10, 20}, pool.get()), + CreateNestedRow(100, std::numeric_limits::quiet_NaN())}); + std::unique_ptr right = + GenericRow::Of({CreateNullableIntArray({1, 9, 3}, /*null_pos=*/1, pool.get()), + CreateIntMap({1, 2}, {10, 20}, pool.get()), + CreateNestedRow(100, -std::numeric_limits::quiet_NaN())}); + ASSERT_EQ(0, equalizer(*left, *right)); + + right->SetField(/*pos=*/0, CreateIntArray({1, 2}, pool.get())); + ASSERT_NE(0, equalizer(*left, *right)); + + right->SetField(/*pos=*/0, CreateNullableIntArray({1, 0, 3}, /*null_pos=*/1, pool.get())); + right->SetField(/*pos=*/0, CreateNullableIntArray({1, 0, 4}, /*null_pos=*/1, pool.get())); + ASSERT_NE(0, equalizer(*left, *right)); + + right->SetField(/*pos=*/0, CreateNullableIntArray({1, 0, 3}, /*null_pos=*/1, pool.get())); + right->SetField(/*pos=*/1, CreateIntMap({1, 3}, {10, 20}, pool.get())); + ASSERT_NE(0, equalizer(*left, *right)); + + right->SetField(/*pos=*/1, CreateIntMap({1, 2}, {10, 20}, pool.get())); + right->SetField(/*pos=*/2, CreateNestedRow(101, 1.0)); + ASSERT_NE(0, equalizer(*left, *right)); +} + +TEST(InternalRowEqualizerTest, MapEqualityDoesNotDependOnEntryOrder) { + std::shared_ptr pool = GetDefaultPool(); + std::shared_ptr schema = + arrow::schema({arrow::field("map", arrow::map(arrow::int32(), arrow::int32()))}); + ASSERT_OK_AND_ASSIGN(FieldsComparator::FieldComparatorFunc equalizer, + InternalRowEqualizer::Create(schema, {})); + + std::unique_ptr left = + GenericRow::Of({CreateIntMap({1, 2, 3}, {10, 20, 30}, pool.get())}); + std::unique_ptr reordered = + GenericRow::Of({CreateIntMap({3, 1, 2}, {30, 10, 20}, pool.get())}); + ASSERT_EQ(0, equalizer(*left, *reordered)); + + reordered->SetField(/*pos=*/0, CreateIntMap({3, 1, 2}, {30, 10, 21}, pool.get())); + ASSERT_NE(0, equalizer(*left, *reordered)); +} + +TEST(InternalRowEqualizerTest, UnsupportedType) { + ASSERT_NOK_WITH_MSG(InternalRowEqualizer::Create( + arrow::schema({arrow::field("unsupported", arrow::null())}), {}), + "Do not support equality for type null"); +} + +} // namespace paimon::test diff --git a/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper.h b/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper.h index d2b21b2c0..34f7cf970 100644 --- a/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper.h +++ b/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper.h @@ -25,9 +25,11 @@ #include #include +#include "paimon/common/data/serializer/row_compacted_serializer.h" #include "paimon/common/utils/fields_comparator.h" #include "paimon/core/deletionvectors/bucketed_dv_maintainer.h" #include "paimon/core/key_value.h" +#include "paimon/core/mergetree/compact/changelog_result.h" #include "paimon/core/mergetree/compact/lookup_merge_function.h" #include "paimon/core/mergetree/compact/merge_function_wrapper.h" #include "paimon/core/mergetree/lookup/file_position.h" @@ -47,23 +49,28 @@ namespace paimon { /// should be AFTER. /// With level-0 record, without level-x record, need to lookup the history value of the upper /// level as BEFORE. -/// TODO(xinyu.lxy) : add changelog template -class LookupChangelogMergeFunctionWrapper : public MergeFunctionWrapper { +class LookupChangelogMergeFunctionWrapper : public MergeFunctionWrapper { public: static Result> Create( std::unique_ptr&& merge_function, std::function>(const std::shared_ptr&)> lookup, - const LookupStrategy& lookup_strategy, + const LookupStrategy& lookup_strategy, bool should_produce_changelog, const std::shared_ptr& deletion_vectors_maintainer, - const std::shared_ptr& comparator) { + const std::shared_ptr& comparator, + std::unique_ptr&& value_serializer, + FieldsComparator::FieldComparatorFunc value_equalizer) { if (lookup_strategy.deletion_vector && !deletion_vectors_maintainer) { return Status::Invalid("deletionVectorsMaintainer should not be null, there is a bug."); } + if (should_produce_changelog && !value_serializer) { + return Status::Invalid("valueSerializer is required when producing changelog."); + } return std::unique_ptr( - new LookupChangelogMergeFunctionWrapper(std::move(merge_function), std::move(lookup), - lookup_strategy, deletion_vectors_maintainer, - comparator)); + new LookupChangelogMergeFunctionWrapper( + std::move(merge_function), std::move(lookup), lookup_strategy, + should_produce_changelog, deletion_vectors_maintainer, comparator, + std::move(value_serializer), std::move(value_equalizer))); } void Reset() override { merge_function_->Reset(); @@ -73,12 +80,17 @@ class LookupChangelogMergeFunctionWrapper : public MergeFunctionWrapperAdd(std::move(kv)); } - Result> GetResult() override { + Result> GetResult() override { // 1. Find the latest high level record and compute containLevel0 - std::optional high_level_idx = merge_function_->PickHighLevelIdx(); + const KeyValue* high_level = merge_function_->PickHighLevel(); + bool contain_level0 = merge_function_->ContainLevel0(); + std::optional before; + if (contain_level0 && should_produce_changelog_ && high_level != nullptr) { + PAIMON_ASSIGN_OR_RAISE(before, CloneKeyValue(*high_level, high_level->value_kind)); + } // 2. Lookup if latest high level record is absent - if (high_level_idx == std::nullopt) { + if (high_level == nullptr) { std::optional lookup_high_level; PAIMON_ASSIGN_OR_RAISE(std::optional lookup_result, lookup_(merge_function_->GetKey())); @@ -105,30 +117,83 @@ class LookupChangelogMergeFunctionWrapper : public MergeFunctionWrapperInsertInto(std::move(lookup_high_level), comparator_); } } // 3. Calculate result PAIMON_ASSIGN_OR_RAISE(std::optional result, merge_function_->GetResult()); - Reset(); + // 4. Set changelog when there's level-0 records - // TODO(liancheng.lsz): setChangelog - return result; + ChangelogResult changelog_result; + if (contain_level0 && should_produce_changelog_) { + PAIMON_RETURN_NOT_OK( + SetChangelog(std::move(before), result, &changelog_result.changelogs)); + } + changelog_result.result = std::move(result); + Reset(); + return std::optional(std::move(changelog_result)); } private: LookupChangelogMergeFunctionWrapper( std::unique_ptr&& merge_function, std::function>(const std::shared_ptr&)> lookup, - const LookupStrategy& lookup_strategy, + const LookupStrategy& lookup_strategy, bool should_produce_changelog, const std::shared_ptr& deletion_vectors_maintainer, - const std::shared_ptr& user_defined_seq_comparator) + const std::shared_ptr& user_defined_seq_comparator, + std::unique_ptr&& value_serializer, + FieldsComparator::FieldComparatorFunc value_equalizer) : merge_function_(std::move(merge_function)), lookup_(std::move(lookup)), lookup_strategy_(lookup_strategy), + should_produce_changelog_(should_produce_changelog), deletion_vectors_maintainer_(deletion_vectors_maintainer), - comparator_(CreateSequenceComparator(user_defined_seq_comparator)) {} + comparator_(CreateSequenceComparator(user_defined_seq_comparator)), + value_serializer_(std::move(value_serializer)), + value_equalizer_(std::move(value_equalizer)) {} + + Result CloneKeyValue(const KeyValue& from, const RowKind* value_kind) { + // TODO(lisizhuo.lsz): avoid serialize & deserialize here. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr bytes, + value_serializer_->SerializeToBytes(*from.value)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr value, + value_serializer_->Deserialize(bytes)); + return KeyValue(value_kind, from.sequence_number, KeyValue::UNKNOWN_LEVEL, from.key, + std::move(value)); + } + + Status SetChangelog(std::optional&& before, const std::optional& after, + std::vector* changelogs) { + if (!before || !before->value_kind->IsAdd()) { + if (after && after->value_kind->IsAdd()) { + PAIMON_ASSIGN_OR_RAISE(KeyValue insert, + CloneKeyValue(after.value(), RowKind::Insert())); + changelogs->emplace_back(std::move(insert)); + } + return Status::OK(); + } + + if (!after || !after->value_kind->IsAdd()) { + before->value_kind = RowKind::Delete(); + changelogs->emplace_back(std::move(before.value())); + return Status::OK(); + } + + if (!value_equalizer_ || value_equalizer_(*before->value, *after->value) != 0) { + before->value_kind = RowKind::UpdateBefore(); + PAIMON_ASSIGN_OR_RAISE(KeyValue update_after, + CloneKeyValue(after.value(), RowKind::UpdateAfter())); + changelogs->emplace_back(std::move(before.value())); + changelogs->emplace_back(std::move(update_after)); + } + return Status::OK(); + } static std::function CreateSequenceComparator( const std::shared_ptr& user_defined_seq_comparator) { @@ -150,8 +215,11 @@ class LookupChangelogMergeFunctionWrapper : public MergeFunctionWrapper merge_function_; std::function>(const std::shared_ptr&)> lookup_; LookupStrategy lookup_strategy_; + bool should_produce_changelog_; std::shared_ptr deletion_vectors_maintainer_; std::function comparator_; + std::unique_ptr value_serializer_; + FieldsComparator::FieldComparatorFunc value_equalizer_; }; } // namespace paimon diff --git a/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper_test.cpp b/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper_test.cpp index 446beb091..c4d255d9a 100644 --- a/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper_test.cpp +++ b/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper_test.cpp @@ -26,12 +26,22 @@ #include "paimon/core/deletionvectors/bucketed_dv_maintainer.h" #include "paimon/core/mergetree/compact/aggregate/aggregate_merge_function.h" #include "paimon/core/mergetree/compact/deduplicate_merge_function.h" +#include "paimon/core/mergetree/compact/internal_row_equalizer.h" #include "paimon/memory/memory_pool.h" #include "paimon/testing/utils/binary_row_generator.h" #include "paimon/testing/utils/key_value_checker.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { +namespace { +std::unique_ptr CreateValueSerializer( + const std::shared_ptr& pool) { + return RowCompactedSerializer::Create(arrow::schema({arrow::field("value", arrow::int32())}), + pool) + .value(); +} +} // namespace + TEST(LookupChangelogMergeFunctionWrapperTest, TestCreateInvalid) { auto pool = GetDefaultPool(); auto mfunc = std::make_unique(/*ignore_delete=*/true); @@ -45,8 +55,10 @@ TEST(LookupChangelogMergeFunctionWrapperTest, TestCreateInvalid) { /*deletion_vector=*/true, /*force_lookup=*/true); ASSERT_NOK_WITH_MSG(LookupChangelogMergeFunctionWrapper::Create( std::move(lookup_mfunc), lookup, lookup_strategy, + /*should_produce_changelog=*/false, /*deletion_vectors_maintainer=*/nullptr, - /*comparator=*/nullptr), + /*comparator=*/nullptr, CreateValueSerializer(pool), + /*value_equalizer=*/{}), "deletionVectorsMaintainer should not be null, there is a bug."); } @@ -69,12 +81,15 @@ TEST(LookupChangelogMergeFunctionWrapperTest, TestSimple) { KeyValue(RowKind::Insert(), /*sequence_number=*/1000, /*level=*/3, key, BinaryRowGenerator::GenerateRowPtr({1001}, pool.get()))); }; - LookupStrategy lookup_strategy(/*is_first_row=*/false, /*produce_changelog=*/false, + LookupStrategy lookup_strategy(/*is_first_row=*/false, /*produce_changelog=*/true, /*deletion_vector=*/false, /*force_lookup=*/true); ASSERT_OK_AND_ASSIGN(auto wrapper, LookupChangelogMergeFunctionWrapper::Create( std::move(lookup_mfunc), lookup, lookup_strategy, + /*should_produce_changelog=*/false, /*deletion_vectors_maintainer=*/nullptr, - /*comparator=*/nullptr)); + /*comparator=*/nullptr, + /*value_serializer=*/nullptr, + /*value_equalizer=*/{})); wrapper->Reset(); ASSERT_OK(wrapper->Add(std::move(kv1))); @@ -82,7 +97,9 @@ TEST(LookupChangelogMergeFunctionWrapperTest, TestSimple) { ASSERT_OK(wrapper->Add(std::move(kv3))); ASSERT_OK_AND_ASSIGN(auto result, wrapper->GetResult()); ASSERT_TRUE(result); - ASSERT_EQ(result.value().sequence_number, 2); + ASSERT_TRUE(result->result); + ASSERT_EQ(result->result->sequence_number, 2); + ASSERT_TRUE(result->changelogs.empty()); } TEST(LookupChangelogMergeFunctionWrapperTest, TestWithLookup) { @@ -104,12 +121,14 @@ TEST(LookupChangelogMergeFunctionWrapperTest, TestWithLookup) { KeyValue(RowKind::Insert(), /*sequence_number=*/0, /*level=*/3, key, BinaryRowGenerator::GenerateRowPtr({1001}, pool.get()))); }; - LookupStrategy lookup_strategy(/*is_first_row=*/false, /*produce_changelog=*/false, + LookupStrategy lookup_strategy(/*is_first_row=*/false, /*produce_changelog=*/true, /*deletion_vector=*/false, /*force_lookup=*/true); ASSERT_OK_AND_ASSIGN(auto wrapper, LookupChangelogMergeFunctionWrapper::Create( std::move(lookup_mfunc), lookup, lookup_strategy, + /*should_produce_changelog=*/true, /*deletion_vectors_maintainer=*/nullptr, - /*comparator=*/nullptr)); + /*comparator=*/nullptr, CreateValueSerializer(pool), + /*value_equalizer=*/{})); wrapper->Reset(); ASSERT_OK(wrapper->Add(std::move(kv1))); @@ -117,7 +136,98 @@ TEST(LookupChangelogMergeFunctionWrapperTest, TestWithLookup) { ASSERT_OK(wrapper->Add(std::move(kv3))); ASSERT_OK_AND_ASSIGN(auto result, wrapper->GetResult()); ASSERT_TRUE(result); - ASSERT_EQ(result.value().sequence_number, 3); + ASSERT_TRUE(result->result); + ASSERT_EQ(result->result->sequence_number, 3); + ASSERT_EQ(result->changelogs.size(), 2); + ASSERT_EQ(result->changelogs[0].value_kind, RowKind::UpdateBefore()); + ASSERT_EQ(result->changelogs[0].level, KeyValue::UNKNOWN_LEVEL); + ASSERT_EQ(result->changelogs[0].value->GetInt(0), 1001); + ASSERT_EQ(result->changelogs[1].value_kind, RowKind::UpdateAfter()); + ASSERT_EQ(result->changelogs[1].level, KeyValue::UNKNOWN_LEVEL); + ASSERT_EQ(result->changelogs[1].value->GetInt(0), 300); +} + +TEST(LookupChangelogMergeFunctionWrapperTest, TestRowDeduplicate) { + auto pool = GetDefaultPool(); + KeyValue kv(RowKind::Insert(), /*sequence_number=*/1, /*level=*/0, + BinaryRowGenerator::GenerateRowPtr({10}, pool.get()), + BinaryRowGenerator::GenerateRowPtr({300}, pool.get())); + + auto merge_function = std::make_unique(/*ignore_delete=*/true); + auto lookup_merge_function = std::make_unique(std::move(merge_function)); + auto lookup = [&](const std::shared_ptr& key) -> Result> { + return std::optional( + KeyValue(RowKind::Insert(), /*sequence_number=*/0, /*level=*/3, key, + BinaryRowGenerator::GenerateRowPtr({300}, pool.get()))); + }; + LookupStrategy lookup_strategy(/*is_first_row=*/false, /*produce_changelog=*/true, + /*deletion_vector=*/false, /*force_lookup=*/true); + auto value_equalizer = [](const InternalRow& lhs, const InternalRow& rhs) { + return lhs.GetInt(0) == rhs.GetInt(0) ? 0 : 1; + }; + ASSERT_OK_AND_ASSIGN(auto wrapper, + LookupChangelogMergeFunctionWrapper::Create( + std::move(lookup_merge_function), lookup, lookup_strategy, + /*should_produce_changelog=*/true, + /*deletion_vectors_maintainer=*/nullptr, + /*comparator=*/nullptr, CreateValueSerializer(pool), value_equalizer)); + + wrapper->Reset(); + ASSERT_OK(wrapper->Add(std::move(kv))); + ASSERT_OK_AND_ASSIGN(auto result, wrapper->GetResult()); + ASSERT_TRUE(result); + ASSERT_TRUE(result->result); + ASSERT_TRUE(result->changelogs.empty()); +} + +TEST(LookupChangelogMergeFunctionWrapperTest, TestRowDeduplicateWithIgnoreFields) { + auto pool = GetDefaultPool(); + auto value_schema = arrow::schema( + {arrow::field("value", arrow::int32()), arrow::field("ignored", arrow::int32())}); + auto merge_function = std::make_unique(/*ignore_delete=*/true); + auto lookup_merge_function = std::make_unique(std::move(merge_function)); + auto lookup = [&](const std::shared_ptr& key) -> Result> { + return std::optional( + KeyValue(RowKind::Insert(), /*sequence_number=*/0, /*level=*/3, key, + BinaryRowGenerator::GenerateRowPtr({300, 1}, pool.get()))); + }; + LookupStrategy lookup_strategy(/*is_first_row=*/false, /*produce_changelog=*/true, + /*deletion_vector=*/false, /*force_lookup=*/true); + ASSERT_OK_AND_ASSIGN(auto value_equalizer, + InternalRowEqualizer::Create(value_schema, {"ignored"})); + ASSERT_OK_AND_ASSIGN(auto value_serializer, RowCompactedSerializer::Create(value_schema, pool)); + ASSERT_OK_AND_ASSIGN(auto wrapper, + LookupChangelogMergeFunctionWrapper::Create( + std::move(lookup_merge_function), lookup, lookup_strategy, + /*should_produce_changelog=*/true, + /*deletion_vectors_maintainer=*/nullptr, + /*comparator=*/nullptr, std::move(value_serializer), value_equalizer)); + + // A change to an ignored field does not produce an update changelog. + wrapper->Reset(); + ASSERT_OK(wrapper->Add(KeyValue(RowKind::Insert(), /*sequence_number=*/1, /*level=*/0, + BinaryRowGenerator::GenerateRowPtr({10}, pool.get()), + BinaryRowGenerator::GenerateRowPtr({300, 2}, pool.get())))); + ASSERT_OK_AND_ASSIGN(auto ignored_field_result, wrapper->GetResult()); + ASSERT_TRUE(ignored_field_result); + ASSERT_TRUE(ignored_field_result->result); + ASSERT_TRUE(ignored_field_result->changelogs.empty()); + + // A change to a non-ignored field still produces update-before and update-after. + wrapper->Reset(); + ASSERT_OK(wrapper->Add(KeyValue(RowKind::Insert(), /*sequence_number=*/2, /*level=*/0, + BinaryRowGenerator::GenerateRowPtr({20}, pool.get()), + BinaryRowGenerator::GenerateRowPtr({301, 2}, pool.get())))); + ASSERT_OK_AND_ASSIGN(auto value_field_result, wrapper->GetResult()); + ASSERT_TRUE(value_field_result); + ASSERT_TRUE(value_field_result->result); + ASSERT_EQ(value_field_result->changelogs.size(), 2); + ASSERT_EQ(value_field_result->changelogs[0].value_kind, RowKind::UpdateBefore()); + ASSERT_EQ(value_field_result->changelogs[0].value->GetInt(0), 300); + ASSERT_EQ(value_field_result->changelogs[0].value->GetInt(1), 1); + ASSERT_EQ(value_field_result->changelogs[1].value_kind, RowKind::UpdateAfter()); + ASSERT_EQ(value_field_result->changelogs[1].value->GetInt(0), 301); + ASSERT_EQ(value_field_result->changelogs[1].value->GetInt(1), 2); } TEST(LookupChangelogMergeFunctionWrapperTest, TestWithLookupWithDv) { @@ -154,8 +264,66 @@ TEST(LookupChangelogMergeFunctionWrapperTest, TestWithLookupWithDv) { /*deletion_vector=*/true, /*force_lookup=*/false); ASSERT_OK_AND_ASSIGN(auto wrapper, LookupChangelogMergeFunctionWrapper::Create( - std::move(lookup_mfunc), lookup, lookup_strategy, dv_maintainer, - /*comparator=*/nullptr)); + std::move(lookup_mfunc), lookup, lookup_strategy, + /*should_produce_changelog=*/false, dv_maintainer, + /*comparator=*/nullptr, + /*value_serializer=*/nullptr, + /*value_equalizer=*/{})); + + wrapper->Reset(); + ASSERT_OK(wrapper->Add(std::move(kv1))); + ASSERT_OK(wrapper->Add(std::move(kv2))); + ASSERT_OK(wrapper->Add(std::move(kv3))); + ASSERT_OK_AND_ASSIGN(auto result, wrapper->GetResult()); + ASSERT_TRUE(result); + ASSERT_TRUE(result->result); + ASSERT_EQ(result->result->sequence_number, 3); + ASSERT_EQ(result->result->value->GetInt(0), 100 + 200 + 300 + 1001); + + auto dv = dv_maintainer->DeletionVectorOf("data.file"); + ASSERT_TRUE(dv); + ASSERT_FALSE(dv.value()->IsDeleted(0).value()); + ASSERT_TRUE(dv.value()->IsDeleted(10).value()); +} + +TEST(LookupChangelogMergeFunctionWrapperTest, TestWithLookupWithDvAndChangelog) { + auto pool = GetDefaultPool(); + KeyValue kv1(RowKind::Insert(), /*sequence_number=*/1, /*level=*/0, /*key=*/ + BinaryRowGenerator::GenerateRowPtr({10}, pool.get()), + /*value=*/BinaryRowGenerator::GenerateRowPtr({100}, pool.get())); + KeyValue kv2(RowKind::Insert(), /*sequence_number=*/2, /*level=*/0, + /*key=*/BinaryRowGenerator::GenerateRowPtr({10}, pool.get()), + /*value=*/BinaryRowGenerator::GenerateRowPtr({200}, pool.get())); + KeyValue kv3(RowKind::Insert(), /*sequence_number=*/3, /*level=*/0, + /*key=*/BinaryRowGenerator::GenerateRowPtr({10}, pool.get()), + /*value=*/BinaryRowGenerator::GenerateRowPtr({300}, pool.get())); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, + CoreOptions::FromMap({{Options::FIELDS_DEFAULT_AGG_FUNC, "sum"}})); + ASSERT_OK_AND_ASSIGN(auto mfunc, AggregateMergeFunction::Create( + arrow::schema({arrow::field("value", arrow::int32())}), + {"key"}, core_options, pool)); + auto lookup_mfunc = std::make_unique(std::move(mfunc)); + auto lookup = + [&](const std::shared_ptr& key) -> Result> { + return std::optional( + {KeyValue(RowKind::Insert(), /*sequence_number=*/0, /*level=*/3, key, + BinaryRowGenerator::GenerateRowPtr({1001}, pool.get())), + "data.file", /*row_position=*/10}); + }; + auto dv_index_file = + std::make_shared(/*fs=*/nullptr, /*path_factory=*/nullptr, + /*bitmap64=*/false, pool); + std::map> deletion_vectors; + auto dv_maintainer = std::make_shared(dv_index_file, deletion_vectors); + + LookupStrategy lookup_strategy(/*is_first_row=*/false, /*produce_changelog=*/true, + /*deletion_vector=*/true, /*force_lookup=*/false); + ASSERT_OK_AND_ASSIGN(auto wrapper, + LookupChangelogMergeFunctionWrapper::Create( + std::move(lookup_mfunc), lookup, lookup_strategy, + /*should_produce_changelog=*/true, dv_maintainer, + /*comparator=*/nullptr, CreateValueSerializer(pool), + /*value_equalizer=*/{})); wrapper->Reset(); ASSERT_OK(wrapper->Add(std::move(kv1))); @@ -163,8 +331,16 @@ TEST(LookupChangelogMergeFunctionWrapperTest, TestWithLookupWithDv) { ASSERT_OK(wrapper->Add(std::move(kv3))); ASSERT_OK_AND_ASSIGN(auto result, wrapper->GetResult()); ASSERT_TRUE(result); - ASSERT_EQ(result.value().sequence_number, 3); - ASSERT_EQ(result.value().value->GetInt(0), 100 + 200 + 300 + 1001); + ASSERT_TRUE(result->result); + ASSERT_EQ(result->result->sequence_number, 3); + ASSERT_EQ(result->result->value->GetInt(0), 100 + 200 + 300 + 1001); + ASSERT_EQ(result->changelogs.size(), 2); + ASSERT_EQ(result->changelogs[0].value_kind, RowKind::UpdateBefore()); + ASSERT_EQ(result->changelogs[0].sequence_number, 0); + ASSERT_EQ(result->changelogs[0].value->GetInt(0), 1001); + ASSERT_EQ(result->changelogs[1].value_kind, RowKind::UpdateAfter()); + ASSERT_EQ(result->changelogs[1].sequence_number, 3); + ASSERT_EQ(result->changelogs[1].value->GetInt(0), 100 + 200 + 300 + 1001); auto dv = dv_maintainer->DeletionVectorOf("data.file"); ASSERT_TRUE(dv); diff --git a/src/paimon/core/mergetree/compact/lookup_merge_function.h b/src/paimon/core/mergetree/compact/lookup_merge_function.h index e50d85eb2..7d8ee1a02 100644 --- a/src/paimon/core/mergetree/compact/lookup_merge_function.h +++ b/src/paimon/core/mergetree/compact/lookup_merge_function.h @@ -100,6 +100,11 @@ class LookupMergeFunction : public MergeFunction { return high_level_idx; } + const KeyValue* PickHighLevel() const { + std::optional high_level_idx = PickHighLevelIdx(); + return high_level_idx ? &candidates_[high_level_idx.value()] : nullptr; + } + private: std::unique_ptr merge_function_; std::vector candidates_; diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp index 071456e48..1a818dd01 100644 --- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp +++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp @@ -23,8 +23,10 @@ #include "paimon/common/table/special_fields.h" #include "paimon/core/mergetree/compact/first_row_merge_function_wrapper.h" #include "paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper.h" +#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h" #include "paimon/core/mergetree/lookup/file_position.h" #include "paimon/core/mergetree/lookup/positioned_key_value.h" +#include "paimon/core/utils/primary_key_table_utils.h" namespace paimon { template @@ -38,7 +40,8 @@ LookupMergeTreeCompactRewriter::LookupMergeTreeCompactRewriter( const std::shared_ptr& path_factory_cache, std::unique_ptr&& merge_file_split_read, MergeFunctionWrapperFactory merge_function_wrapper_factory, - const std::shared_ptr& cancellation_controller, + ChangelogMergeFunctionWrapperFactory changelog_merge_function_wrapper_factory, + bool produce_changelog, const std::shared_ptr& cancellation_controller, const std::shared_ptr& remote_lookup_file_manager, const std::shared_ptr& pool) : ChangelogMergeTreeRewriter( @@ -46,6 +49,7 @@ LookupMergeTreeCompactRewriter::LookupMergeTreeCompactRewriter( trimmed_primary_keys, options, data_schema, write_schema, DeletionVector::CreateFactory(dv_maintainer), path_factory_cache, std::move(merge_file_split_read), std::move(merge_function_wrapper_factory), + std::move(changelog_merge_function_wrapper_factory), produce_changelog, cancellation_controller, pool), lookup_levels_(std::move(lookup_levels)), dv_maintainer_(dv_maintainer), @@ -56,10 +60,10 @@ Result>> LookupMergeTreeCompactRewriter::Create( int32_t max_level, std::unique_ptr>&& lookup_levels, const std::shared_ptr& dv_maintainer, - MergeFunctionWrapperFactory merge_function_wrapper_factory, int32_t bucket, + ChangelogMergeFunctionWrapperFactory changelog_merge_function_wrapper_factory, int32_t bucket, const BinaryRow& partition, const std::shared_ptr& table_schema, const std::shared_ptr& path_factory_cache, - const CoreOptions& options, + const CoreOptions& options, bool produce_changelog, const std::shared_ptr& cancellation_controller, const std::shared_ptr& remote_lookup_file_manager, const std::shared_ptr& pool) { @@ -88,44 +92,60 @@ LookupMergeTreeCompactRewriter::Create( std::unique_ptr merge_file_split_read, MergeFileSplitRead::Create(path_factory, internal_context, pool, CreateDefaultExecutor())); + MergeFunctionWrapperFactory merge_function_wrapper_factory = + [data_schema, options, trimmed_primary_keys, pool]( + int32_t /*output_level*/) -> Result>> { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merge_function, + PrimaryKeyTableUtils::CreateMergeFunction( + data_schema, trimmed_primary_keys, options, pool)); + if (options.NeedLookup() && options.GetMergeEngine() != MergeEngine::FIRST_ROW) { + merge_function = std::make_unique(std::move(merge_function)); + } + return std::make_shared(std::move(merge_function)); + }; + return std::unique_ptr(new LookupMergeTreeCompactRewriter( std::move(lookup_levels), dv_maintainer, max_level, partition, bucket, table_schema->Id(), trimmed_primary_keys, options, data_schema, write_schema, path_factory_cache, std::move(merge_file_split_read), std::move(merge_function_wrapper_factory), + std::move(changelog_merge_function_wrapper_factory), produce_changelog, cancellation_controller, remote_lookup_file_manager, pool)); } template -std::shared_ptr> +std::shared_ptr> LookupMergeTreeCompactRewriter::CreateFirstRowMergeFunctionWrapper( std::unique_ptr&& merge_func, int32_t output_level, - LookupLevels* lookup_levels) { + std::unique_ptr&& value_serializer, LookupLevels* lookup_levels) { auto contains = [output_level, lookup_levels](const std::shared_ptr& key) -> Result { PAIMON_ASSIGN_OR_RAISE(std::optional contain, lookup_levels->Lookup(key, output_level + 1)); return contain != std::nullopt; }; - return std::make_shared(std::move(merge_func), - std::move(contains)); + return std::make_shared( + std::move(merge_func), std::move(contains), std::move(value_serializer)); } template -Result>> +Result>> LookupMergeTreeCompactRewriter::CreateLookupMergeFunctionWrapper( std::unique_ptr&& merge_func, int32_t output_level, const std::shared_ptr& deletion_vectors_maintainer, - const LookupStrategy& lookup_strategy, + const LookupStrategy& lookup_strategy, bool should_produce_changelog, const std::shared_ptr& user_defined_seq_comparator, - LookupLevels* lookup_levels) { + std::unique_ptr&& value_serializer, + FieldsComparator::FieldComparatorFunc value_equalizer, LookupLevels* lookup_levels) { auto lookup = [output_level, lookup_levels]( const std::shared_ptr& key) -> Result> { return lookup_levels->Lookup(key, output_level + 1); }; - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr> wrapper, - LookupChangelogMergeFunctionWrapper::Create( - std::move(merge_func), std::move(lookup), lookup_strategy, - deletion_vectors_maintainer, user_defined_seq_comparator)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr> wrapper, + LookupChangelogMergeFunctionWrapper::Create( + std::move(merge_func), std::move(lookup), lookup_strategy, should_produce_changelog, + deletion_vectors_maintainer, user_defined_seq_comparator, std::move(value_serializer), + std::move(value_equalizer))); return wrapper; } diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h index 06c5f47f4..a461177ba 100644 --- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h +++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h @@ -19,10 +19,12 @@ #pragma once #include "arrow/api.h" +#include "paimon/common/data/serializer/row_compacted_serializer.h" #include "paimon/core/core_options.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/core/mergetree/compact/changelog_merge_tree_rewriter.h" #include "paimon/core/mergetree/compact/first_row_merge_function.h" +#include "paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper.h" #include "paimon/core/mergetree/compact/lookup_merge_function.h" #include "paimon/core/mergetree/lookup/remote_lookup_file_manager.h" #include "paimon/core/mergetree/lookup_levels.h" @@ -38,10 +40,11 @@ class LookupMergeTreeCompactRewriter : public ChangelogMergeTreeRewriter { static Result> Create( int32_t max_level, std::unique_ptr>&& lookup_levels, const std::shared_ptr& dv_maintainer, - MergeFunctionWrapperFactory merge_function_wrapper_factory, int32_t bucket, - const BinaryRow& partition, const std::shared_ptr& table_schema, + ChangelogMergeFunctionWrapperFactory changelog_merge_function_wrapper_factory, + int32_t bucket, const BinaryRow& partition, + const std::shared_ptr& table_schema, const std::shared_ptr& path_factory_cache, - const CoreOptions& options, + const CoreOptions& options, bool produce_changelog, const std::shared_ptr& cancellation_controller, const std::shared_ptr& remote_lookup_file_manager, const std::shared_ptr& pool); @@ -50,16 +53,20 @@ class LookupMergeTreeCompactRewriter : public ChangelogMergeTreeRewriter { return lookup_levels_->Close(); } - static std::shared_ptr> CreateFirstRowMergeFunctionWrapper( - std::unique_ptr&& merge_func, int32_t output_level, - LookupLevels* lookup_levels); + static std::shared_ptr> + CreateFirstRowMergeFunctionWrapper(std::unique_ptr&& merge_func, + int32_t output_level, + std::unique_ptr&& value_serializer, + LookupLevels* lookup_levels); - static Result>> CreateLookupMergeFunctionWrapper( + static Result>> + CreateLookupMergeFunctionWrapper( std::unique_ptr&& merge_func, int32_t output_level, const std::shared_ptr& deletion_vectors_maintainer, - const LookupStrategy& lookup_strategy, + const LookupStrategy& lookup_strategy, bool should_produce_changelog, const std::shared_ptr& user_defined_seq_comparator, - LookupLevels* lookup_levels); + std::unique_ptr&& value_serializer, + FieldsComparator::FieldComparatorFunc value_equalizer, LookupLevels* lookup_levels); private: LookupMergeTreeCompactRewriter( @@ -72,6 +79,8 @@ class LookupMergeTreeCompactRewriter : public ChangelogMergeTreeRewriter { const std::shared_ptr& path_factory_cache, std::unique_ptr&& merge_file_split_read, MergeFunctionWrapperFactory merge_function_wrapper_factory, + ChangelogMergeFunctionWrapperFactory changelog_merge_function_wrapper_factory, + bool produce_changelog, const std::shared_ptr& cancellation_controller, const std::shared_ptr& remote_lookup_file_manager, const std::shared_ptr& pool); diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp index 169170da1..c930c5ca7 100644 --- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp +++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp @@ -24,6 +24,7 @@ #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/utils/fields_comparator.h" @@ -35,6 +36,7 @@ #include "paimon/core/mergetree/compact/aggregate/aggregate_merge_function.h" #include "paimon/core/mergetree/compact/changelog_merge_tree_rewriter.h" #include "paimon/core/mergetree/compact/deduplicate_merge_function.h" +#include "paimon/core/mergetree/compact/internal_row_equalizer.h" #include "paimon/core/mergetree/compact/interval_partition.h" #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h" #include "paimon/core/mergetree/lookup/default_lookup_serializer_factory.h" @@ -148,16 +150,25 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam>> CreateCompactRewriterForFirstRow( const std::string& table_path, const std::shared_ptr& table_schema, - const CoreOptions& options, std::unique_ptr>&& lookup_levels) const { + const CoreOptions& options, std::unique_ptr>&& lookup_levels, + std::optional produce_changelog = std::nullopt) const { auto path_factory_cache = std::make_shared(table_path, table_schema, options, pool_); + bool should_produce_changelog = + produce_changelog.value_or(options.GetLookupStrategy().produce_changelog); auto merge_function_wrapper_factory = - [lookup_levels_ptr = lookup_levels.get()]( - int32_t output_level) -> Result>> { - std::shared_ptr> merge_function_wrapper = + [lookup_levels_ptr = lookup_levels.get(), data_schema = arrow_schema_, + should_produce_changelog, pool = pool_](int32_t output_level) + -> Result>> { + std::unique_ptr value_serializer; + if (should_produce_changelog) { + PAIMON_ASSIGN_OR_RAISE(value_serializer, + RowCompactedSerializer::Create(data_schema, pool)); + } + std::shared_ptr> merge_function_wrapper = LookupMergeTreeCompactRewriter::CreateFirstRowMergeFunctionWrapper( std::make_unique(/*ignore_delete=*/true), output_level, - lookup_levels_ptr); + std::move(value_serializer), lookup_levels_ptr); return merge_function_wrapper; }; auto cancellation_controller = std::make_shared(); @@ -166,7 +177,8 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam>> @@ -177,18 +189,29 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam(table_path, table_schema, options, pool_); auto merge_function_wrapper_factory = [this, table_schema, options, lookup_levels_ptr = lookup_levels.get(), - lookup_strategy = options.GetLookupStrategy()]( - int32_t output_level) -> Result>> { - PAIMON_ASSIGN_OR_RAISE(auto merge_func, - AggregateMergeFunction::Create( - arrow_schema_, table_schema->TrimmedPrimaryKeys().value(), - options, GetDefaultPool())); + lookup_strategy = options.GetLookupStrategy()](int32_t output_level) + -> Result>> { PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr> merge_function_wrapper, + auto merge_func, + AggregateMergeFunction::Create( + arrow_schema_, table_schema->TrimmedPrimaryKeys().value(), options, pool_)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr value_serializer, + RowCompactedSerializer::Create(arrow_schema_, pool_)); + FieldsComparator::FieldComparatorFunc value_equalizer; + if (options.ChangelogRowDeduplicate()) { + PAIMON_ASSIGN_OR_RAISE( + value_equalizer, + InternalRowEqualizer::Create(arrow_schema_, + options.GetChangelogRowDeduplicateIgnoreFields())); + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr> merge_function_wrapper, LookupMergeTreeCompactRewriter::CreateLookupMergeFunctionWrapper( std::make_unique(std::move(merge_func)), output_level, /*deletion_vectors_maintainer=*/nullptr, lookup_strategy, - /*user_defined_seq_comparator=*/nullptr, lookup_levels_ptr)); + /*should_produce_changelog=*/lookup_strategy.produce_changelog, + /*user_defined_seq_comparator=*/nullptr, std::move(value_serializer), + std::move(value_equalizer), lookup_levels_ptr)); return merge_function_wrapper; }; auto cancellation_controller = std::make_shared(); @@ -196,7 +219,8 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam>> @@ -213,17 +237,22 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam(dv_index_file, deletion_vectors); - auto merge_function_wrapper_factory = - [lookup_levels_ptr = lookup_levels.get(), lookup_strategy = options.GetLookupStrategy(), - dv_maintainer_ptr = dv_maintainer]( - int32_t output_level) -> Result>> { + auto merge_function_wrapper_factory = [this, lookup_levels_ptr = lookup_levels.get(), + lookup_strategy = options.GetLookupStrategy(), + dv_maintainer_ptr = + dv_maintainer](int32_t output_level) + -> Result>> { auto merge_func = std::make_unique(false); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr value_serializer, + RowCompactedSerializer::Create(arrow_schema_, pool_)); PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr> merge_function_wrapper, + std::shared_ptr> merge_function_wrapper, LookupMergeTreeCompactRewriter::CreateLookupMergeFunctionWrapper( std::make_unique(std::move(merge_func)), output_level, dv_maintainer_ptr, lookup_strategy, - /*user_defined_seq_comparator=*/nullptr, lookup_levels_ptr)); + /*should_produce_changelog=*/lookup_strategy.produce_changelog, + /*user_defined_seq_comparator=*/nullptr, std::move(value_serializer), + /*value_equalizer=*/{}, lookup_levels_ptr)); return merge_function_wrapper; }; auto cancellation_controller = std::make_shared(); @@ -231,7 +260,8 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam>> @@ -253,19 +283,31 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam Result>> { - PAIMON_ASSIGN_OR_RAISE(auto merge_func, - AggregateMergeFunction::Create( - arrow_schema_, table_schema->TrimmedPrimaryKeys().value(), - options, GetDefaultPool())); + lookup_strategy = options.GetLookupStrategy(), + dv_maintainer_ptr = dv_maintainer](int32_t output_level) + -> Result>> { PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr> merge_function_wrapper, + auto merge_func, + AggregateMergeFunction::Create( + arrow_schema_, table_schema->TrimmedPrimaryKeys().value(), options, pool_)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr value_serializer, + RowCompactedSerializer::Create(arrow_schema_, pool_)); + FieldsComparator::FieldComparatorFunc value_equalizer; + if (options.ChangelogRowDeduplicate()) { + PAIMON_ASSIGN_OR_RAISE( + value_equalizer, + InternalRowEqualizer::Create(arrow_schema_, + options.GetChangelogRowDeduplicateIgnoreFields())); + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr> merge_function_wrapper, LookupMergeTreeCompactRewriter:: CreateLookupMergeFunctionWrapper( std::make_unique(std::move(merge_func)), output_level, dv_maintainer_ptr, lookup_strategy, - /*user_defined_seq_comparator=*/nullptr, lookup_levels_ptr)); + /*should_produce_changelog=*/lookup_strategy.produce_changelog, + /*user_defined_seq_comparator=*/nullptr, std::move(value_serializer), + std::move(value_equalizer), lookup_levels_ptr)); return merge_function_wrapper; }; auto cancellation_controller = std::make_shared(); @@ -273,7 +315,8 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParamEquals(*result_array)) << result_array->ToString(); } + void CheckShreddingFileSchema(const std::string& file_name, + const std::shared_ptr& table_schema, + const std::string& file_format_name, + const std::shared_ptr& expected_physical_schema, + int32_t field_index, + const MapSharedShreddingFieldMeta& expected_meta) const { + ASSERT_OK_AND_ASSIGN(auto file_format, + FileFormatFactory::Get(file_format_name, table_schema->Options())); + ASSERT_OK_AND_ASSIGN(auto reader_builder, + file_format->CreateReaderBuilder(/*batch_size=*/10)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, fs_->Open(file_name)); + ASSERT_OK_AND_ASSIGN(auto file_batch_reader, reader_builder->Build(input_stream)); + ASSERT_OK_AND_ASSIGN(auto c_file_schema, file_batch_reader->GetFileSchema()); + std::shared_ptr file_schema = + arrow::ImportSchema(c_file_schema.get()).ValueOrDie(); + + ASSERT_TRUE(file_schema->Equals(*expected_physical_schema, /*check_metadata=*/false)) + << "Expected schema:\n" + << expected_physical_schema->ToString() << "\nActual schema:\n" + << file_schema->ToString(); + std::shared_ptr metadata = + file_schema->field(field_index)->metadata(); + ASSERT_NE(nullptr, metadata); + ASSERT_OK_AND_ASSIGN(MapSharedShreddingFieldMeta actual_meta, + MapSharedShreddingUtils::DeserializeMetadata(metadata->Copy())); + ASSERT_EQ(expected_meta, actual_meta); + } + Result> CreateFileStorePathFactory( const std::string& table_path, const CoreOptions& options) const { PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, @@ -422,7 +493,8 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam options = {{Options::MERGE_ENGINE, "first-row"}, - {Options::FILE_FORMAT, "orc"}}; + {Options::FILE_FORMAT, "orc"}, + {Options::CHANGELOG_PRODUCER, "lookup"}}; ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); ASSERT_OK_AND_ASSIGN(auto table_path, CreateTable(options)); auto schema_manager = std::make_shared(fs_, table_path); @@ -447,6 +519,7 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestFirstRowRewrite) { /*output_level=*/5, /*drop_delete=*/true, runs)); ASSERT_EQ(2, compact_result.Before().size()); ASSERT_EQ(1, compact_result.After().size()); + ASSERT_EQ(1, compact_result.Changelog().size()); // check compact result const auto& compact_file_meta = compact_result.After()[0]; @@ -483,11 +556,15 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestFirstRowRewrite) { &expected_array); ASSERT_TRUE(array_status.ok()); CheckResult(compact_file_name, table_schema, "orc", expected_array); + std::string changelog_file_name = + table_path + "/bucket-0/" + compact_result.Changelog()[0]->file_name; + CheckResult(changelog_file_name, table_schema, "orc", expected_array); } TEST_F(LookupMergeTreeCompactRewriterTest, TestFirstRowLooksUpExistingKeys) { std::map options = {{Options::MERGE_ENGINE, "first-row"}, - {Options::FILE_FORMAT, "orc"}}; + {Options::FILE_FORMAT, "orc"}, + {Options::CHANGELOG_PRODUCER, "lookup"}}; ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); ASSERT_OK_AND_ASSIGN(auto table_path, CreateTable(options)); auto schema_manager = std::make_shared(fs_, table_path); @@ -513,6 +590,8 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestFirstRowLooksUpExistingKeys) { ASSERT_EQ(1, compact_result.After().size()); ASSERT_EQ(1, compact_result.After()[0]->row_count); + ASSERT_EQ(1, compact_result.Changelog().size()); + ASSERT_EQ(1, compact_result.Changelog()[0]->row_count); auto type_with_special_fields = arrow::struct_(SpecialFields::CompleteSequenceAndValueKindField(arrow_schema_)->fields()); @@ -522,6 +601,103 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestFirstRowLooksUpExistingKeys) { .ok()); CheckResult(table_path + "/bucket-0/" + compact_result.After()[0]->file_name, table_schema, "orc", expected); + CheckResult(table_path + "/bucket-0/" + compact_result.Changelog()[0]->file_name, table_schema, + "orc", expected); +} + +TEST_F(LookupMergeTreeCompactRewriterTest, TestLookupChangelogCanBeDisabled) { + std::map options = {{Options::MERGE_ENGINE, "first-row"}, + {Options::FILE_FORMAT, "orc"}, + {Options::CHANGELOG_PRODUCER, "lookup"}}; + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); + ASSERT_OK_AND_ASSIGN(auto table_path, CreateTable(options)); + auto schema_manager = std::make_shared(fs_, table_path); + ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager->ReadSchema(0)); + ASSERT_OK_AND_ASSIGN(auto file, NewFiles(/*level=*/0, /*last_sequence_number=*/-1, table_path, + core_options, "[[1, 11], [2, 22]]")); + auto processor_factory = std::make_shared(); + ASSERT_OK_AND_ASSIGN(auto lookup_levels, CreateLookupLevels(table_path, table_schema, + processor_factory, {file})); + ASSERT_OK_AND_ASSIGN(auto rewriter, + CreateCompactRewriterForFirstRow(table_path, table_schema, core_options, + std::move(lookup_levels), + /*produce_changelog=*/false)); + ASSERT_OK_AND_ASSIGN(auto runs, GenerateSortedRuns({file})); + ASSERT_OK_AND_ASSIGN(auto compact_result, rewriter->Rewrite( + /*output_level=*/1, /*drop_delete=*/true, runs)); + + ASSERT_EQ(1, compact_result.After().size()); + ASSERT_TRUE(compact_result.Changelog().empty()); +} + +TEST_F(LookupMergeTreeCompactRewriterTest, TestFirstRowRewriteWithSharedShreddingChangelog) { + arrow::FieldVector fields = { + arrow::field("key", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }; + arrow_schema_ = arrow::schema(fields); + key_schema_ = arrow::schema({fields[0]}); + std::map options = { + {Options::MERGE_ENGINE, "first-row"}, + {Options::FILE_FORMAT, "orc"}, + {Options::CHANGELOG_PRODUCER, "lookup"}, + {Options::CHANGELOG_FILE_FORMAT, "parquet"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "3"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, + }; + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); + ASSERT_OK_AND_ASSIGN(auto table_path, CreateTable(options)); + auto schema_manager = std::make_shared(fs_, table_path); + ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager->ReadSchema(0)); + + ASSERT_OK_AND_ASSIGN( + auto file0, NewFiles(/*level=*/0, /*last_sequence_number=*/-1, table_path, core_options, + R"([[1, [["a", 11]]], [3, [["c", 33]]], [5, [["e", 55]]]])")); + ASSERT_OK_AND_ASSIGN(auto file1, + NewFiles(/*level=*/0, /*last_sequence_number=*/2, table_path, core_options, + R"([[2, [["b", 22]]], [5, [["f", 555]]]])")); + std::vector> files = {file0, file1}; + auto processor_factory = std::make_shared(); + ASSERT_OK_AND_ASSIGN(auto lookup_levels, CreateLookupLevels(table_path, table_schema, + processor_factory, files)); + ASSERT_OK_AND_ASSIGN(auto rewriter, + CreateCompactRewriterForFirstRow(table_path, table_schema, core_options, + std::move(lookup_levels))); + ASSERT_OK_AND_ASSIGN(auto runs, GenerateSortedRuns(files)); + ASSERT_OK_AND_ASSIGN(auto compact_result, rewriter->Rewrite( + /*output_level=*/5, /*drop_delete=*/true, runs)); + + ASSERT_EQ(1, compact_result.Changelog().size()); + const std::shared_ptr& changelog_file = compact_result.Changelog()[0]; + ASSERT_EQ(4, changelog_file->row_count); + ASSERT_EQ(FileSource::Append(), changelog_file->file_source); + ASSERT_TRUE(StringUtils::EndsWith(changelog_file->file_name, ".parquet")); + std::string changelog_file_name = table_path + "/bucket-0/" + changelog_file->file_name; + + std::shared_ptr write_schema = SpecialFields::CompleteSequenceAndValueKindField( + DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields())); + ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + write_schema, {{"tags", 3}})); + std::shared_ptr expected_changelog; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + arrow::struct_(physical_schema->fields()), {R"([ + [0, 0, 1, [[0, -1, -1], 11, null, null, null]], + [3, 0, 2, [[1, -1, -1], 22, null, null, null]], + [1, 0, 3, [[2, -1, -1], 33, null, null, null]], + [2, 0, 5, [[3, -1, -1], 55, null, null, null]] + ])"}, + &expected_changelog) + .ok()); + CheckResult(changelog_file_name, table_schema, "parquet", expected_changelog); + + MapSharedShreddingFieldMeta expected_meta; + expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}, {"e", 3}}; + expected_meta.field_to_columns = {{0, {0}}, {1, {0}}, {2, {0}}, {3, {0}}}; + expected_meta.num_columns = 3; + expected_meta.max_row_width = 1; + CheckShreddingFileSchema(changelog_file_name, table_schema, "parquet", physical_schema, + /*field_index=*/3, expected_meta); } TEST_F(LookupMergeTreeCompactRewriterTest, TestFirstRowUpgrade) { @@ -680,7 +856,7 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestRewriteWithAllHighLevel) { TEST_F(LookupMergeTreeCompactRewriterTest, TestRewriteWithForceLookupAndSumAgg) { std::map options = {{Options::MERGE_ENGINE, "aggregation"}, {Options::FILE_FORMAT, "orc"}, - {Options::FORCE_LOOKUP, "true"}, + {Options::CHANGELOG_PRODUCER, "lookup"}, {Options::FIELDS_DEFAULT_AGG_FUNC, "sum"}}; ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); ASSERT_OK_AND_ASSIGN(auto table_path, CreateTable(options)); @@ -709,6 +885,7 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestRewriteWithForceLookupAndSumAgg) /*output_level=*/4, /*drop_delete=*/true, runs)); ASSERT_EQ(2, compact_result.Before().size()); ASSERT_EQ(1, compact_result.After().size()); + ASSERT_EQ(1, compact_result.Changelog().size()); const auto& compact_file_meta = compact_result.After()[0]; // check compact file exist @@ -729,6 +906,20 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestRewriteWithForceLookupAndSumAgg) &expected_array); ASSERT_TRUE(array_status.ok()); CheckResult(compact_file_name, table_schema, "orc", expected_array); + + const auto& changelog_file_meta = compact_result.Changelog()[0]; + std::string changelog_file_name = table_path + "/bucket-0/" + changelog_file_meta->file_name; + std::shared_ptr expected_changelog; + auto changelog_status = + arrow::ipc::internal::json::ChunkedArrayFromJSON(type_with_special_fields, {R"([ +[6, 0, 2, 244], +[4, 0, 4, 44], +[2, 1, 5, 55], +[7, 2, 5, 615] +])"}, + &expected_changelog); + ASSERT_TRUE(changelog_status.ok()); + CheckResult(changelog_file_name, table_schema, "orc", expected_changelog); } TEST_F(LookupMergeTreeCompactRewriterTest, TestRewriteWithDvAndDeduplicate) { @@ -1122,7 +1313,9 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestGenerateUpgradeStrategy) { /*trimmed_primary_keys=*/{"key"}, core_options, /*data_schema=*/nullptr, /*write_schema=*/nullptr, /*path_factory_cache=*/nullptr, /*merge_file_split_read=*/nullptr, /*merge_function_wrapper_factory=*/nullptr, - cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_); + /*changelog_merge_function_wrapper_factory=*/nullptr, + /*produce_changelog=*/false, cancellation_controller, + /*remote_lookup_file_manager=*/nullptr, pool_); auto file = create_meta(/*level=*/1, /*delete_row_count=*/std::nullopt); ASSERT_EQ(ChangelogMergeTreeRewriter::UpgradeStrategy::NoChangelogNoRewrite(), rewriter.GenerateUpgradeStrategy(/*output_level=*/2, file)); @@ -1137,7 +1330,9 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestGenerateUpgradeStrategy) { /*trimmed_primary_keys=*/{"key"}, core_options, /*data_schema=*/nullptr, /*write_schema=*/nullptr, /*path_factory_cache=*/nullptr, /*merge_file_split_read=*/nullptr, /*merge_function_wrapper_factory=*/nullptr, - cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_); + /*changelog_merge_function_wrapper_factory=*/nullptr, + /*produce_changelog=*/false, cancellation_controller, + /*remote_lookup_file_manager=*/nullptr, pool_); auto file = create_meta(/*level=*/0, /*delete_row_count=*/std::nullopt); ASSERT_EQ(ChangelogMergeTreeRewriter::UpgradeStrategy::ChangelogWithRewrite(), rewriter.GenerateUpgradeStrategy(/*output_level=*/5, file)); @@ -1156,7 +1351,9 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestGenerateUpgradeStrategy) { /*trimmed_primary_keys=*/{"key"}, core_options, /*data_schema=*/nullptr, /*write_schema=*/nullptr, /*path_factory_cache=*/nullptr, /*merge_file_split_read=*/nullptr, /*merge_function_wrapper_factory=*/nullptr, - cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_); + /*changelog_merge_function_wrapper_factory=*/nullptr, + /*produce_changelog=*/false, cancellation_controller, + /*remote_lookup_file_manager=*/nullptr, pool_); auto file = create_meta(/*level=*/0, /*delete_row_count=*/1); ASSERT_EQ(ChangelogMergeTreeRewriter::UpgradeStrategy::ChangelogWithRewrite(), rewriter.GenerateUpgradeStrategy(/*output_level=*/2, file)); @@ -1171,7 +1368,9 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestGenerateUpgradeStrategy) { /*trimmed_primary_keys=*/{"key"}, core_options, /*data_schema=*/nullptr, /*write_schema=*/nullptr, /*path_factory_cache=*/nullptr, /*merge_file_split_read=*/nullptr, /*merge_function_wrapper_factory=*/nullptr, - cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_); + /*changelog_merge_function_wrapper_factory=*/nullptr, + /*produce_changelog=*/false, cancellation_controller, + /*remote_lookup_file_manager=*/nullptr, pool_); auto file = create_meta(/*level=*/0, /*delete_row_count=*/std::nullopt); ASSERT_EQ(ChangelogMergeTreeRewriter::UpgradeStrategy::ChangelogNoRewrite(), rewriter.GenerateUpgradeStrategy(/*output_level=*/5, file)); @@ -1186,7 +1385,9 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestGenerateUpgradeStrategy) { /*trimmed_primary_keys=*/{"key"}, core_options, /*data_schema=*/nullptr, /*write_schema=*/nullptr, /*path_factory_cache=*/nullptr, /*merge_file_split_read=*/nullptr, /*merge_function_wrapper_factory=*/nullptr, - cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_); + /*changelog_merge_function_wrapper_factory=*/nullptr, + /*produce_changelog=*/false, cancellation_controller, + /*remote_lookup_file_manager=*/nullptr, pool_); auto file = create_meta(/*level=*/0, /*delete_row_count=*/std::nullopt); ASSERT_EQ(ChangelogMergeTreeRewriter::UpgradeStrategy::ChangelogNoRewrite(), rewriter.GenerateUpgradeStrategy(/*output_level=*/2, file)); @@ -1202,7 +1403,9 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestGenerateUpgradeStrategy) { /*trimmed_primary_keys=*/{"key"}, core_options, /*data_schema=*/nullptr, /*write_schema=*/nullptr, /*path_factory_cache=*/nullptr, /*merge_file_split_read=*/nullptr, /*merge_function_wrapper_factory=*/nullptr, - cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_); + /*changelog_merge_function_wrapper_factory=*/nullptr, + /*produce_changelog=*/false, cancellation_controller, + /*remote_lookup_file_manager=*/nullptr, pool_); auto file = create_meta(/*level=*/0, /*delete_row_count=*/std::nullopt); ASSERT_EQ(ChangelogMergeTreeRewriter::UpgradeStrategy::ChangelogWithRewrite(), rewriter.GenerateUpgradeStrategy(/*output_level=*/2, file)); diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.cpp index e5cb3abf6..258336698 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.cpp @@ -25,6 +25,7 @@ #include "paimon/core/mergetree/compact/aggregate/aggregate_merge_function.h" #include "paimon/core/mergetree/compact/early_full_compaction.h" #include "paimon/core/mergetree/compact/force_up_level0_compaction.h" +#include "paimon/core/mergetree/compact/internal_row_equalizer.h" #include "paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h" #include "paimon/core/mergetree/compact/merge_tree_compact_manager.h" #include "paimon/core/mergetree/compact/merge_tree_compact_rewriter.h" @@ -50,6 +51,24 @@ namespace paimon { namespace { +Result CreateChangelogValueEqualizer( + const std::shared_ptr& schema, const CoreOptions& options, + bool produce_changelog) { + if (!produce_changelog || !options.ChangelogRowDeduplicate()) { + return FieldsComparator::FieldComparatorFunc(); + } + return InternalRowEqualizer::Create(schema, options.GetChangelogRowDeduplicateIgnoreFields()); +} + +Result> CreateChangelogValueSerializer( + const std::shared_ptr& schema, bool produce_changelog, + const std::shared_ptr& pool) { + if (!produce_changelog) { + return std::unique_ptr(); + } + return RowCompactedSerializer::Create(schema, pool); +} + template Result>> CreateLookupLevelsInternal( const CoreOptions& options, const std::shared_ptr& schema_manager, @@ -174,6 +193,8 @@ Result> MergeTreeCompactManagerFactory::CreateL const LookupStrategy& lookup_strategy, const std::shared_ptr& path_factory_cache, const std::shared_ptr& cancellation_controller) const { + const bool should_produce_changelog = + lookup_strategy.produce_changelog && !ignore_previous_files_; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr remote_lookup_file_manager, CreateRemoteLookupFileManager(partition, bucket)); if (lookup_strategy.is_first_row) { @@ -190,157 +211,133 @@ Result> MergeTreeCompactManagerFactory::CreateL options_, schema_manager_, io_manager_, cache_manager_, file_store_path_factory_, table_schema_, partition, bucket, levels, processor_factory, dv_maintainer, lookup_file_cache_, remote_lookup_file_manager, pool_)); - auto merge_function_wrapper_factory = - [lookup_levels_ptr = lookup_levels.get(), ignore_delete = options_.IgnoreDelete()]( - int32_t output_level) -> Result>> { - std::shared_ptr> merge_function_wrapper = + auto merge_function_wrapper_factory = [lookup_levels_ptr = lookup_levels.get(), + data_schema = schema_, should_produce_changelog, + ignore_delete = options_.IgnoreDelete(), + pool = pool_](int32_t output_level) + -> Result>> { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr value_serializer, + CreateChangelogValueSerializer(data_schema, should_produce_changelog, pool)); + std::shared_ptr> merge_function_wrapper = LookupMergeTreeCompactRewriter::CreateFirstRowMergeFunctionWrapper( std::make_unique(ignore_delete), output_level, - lookup_levels_ptr); + std::move(value_serializer), lookup_levels_ptr); return merge_function_wrapper; }; - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr> rewriter, - LookupMergeTreeCompactRewriter::Create( - max_level, std::move(lookup_levels), dv_maintainer, - std::move(merge_function_wrapper_factory), bucket, partition, - table_schema_, path_factory_cache, options_, - cancellation_controller, remote_lookup_file_manager, pool_)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr> rewriter, + LookupMergeTreeCompactRewriter::Create( + max_level, std::move(lookup_levels), dv_maintainer, + std::move(merge_function_wrapper_factory), bucket, partition, table_schema_, + path_factory_cache, options_, should_produce_changelog, cancellation_controller, + remote_lookup_file_manager, pool_)); return std::shared_ptr(std::move(rewriter)); } if (lookup_strategy.deletion_vector) { return CreateLookupRewriterWithDeletionVector( partition, bucket, levels, dv_maintainer, max_level, lookup_strategy, - path_factory_cache, cancellation_controller, remote_lookup_file_manager); + should_produce_changelog, path_factory_cache, cancellation_controller, + remote_lookup_file_manager); } return CreateLookupRewriterWithoutDeletionVector( - partition, bucket, levels, dv_maintainer, max_level, lookup_strategy, path_factory_cache, - cancellation_controller, remote_lookup_file_manager); + partition, bucket, levels, dv_maintainer, max_level, lookup_strategy, + should_produce_changelog, path_factory_cache, cancellation_controller, + remote_lookup_file_manager); } +template Result> -MergeTreeCompactManagerFactory::CreateLookupRewriterWithDeletionVector( +MergeTreeCompactManagerFactory::CreateLookupRewriterInternal( const BinaryRow& partition, int32_t bucket, const std::shared_ptr& levels, const std::shared_ptr& dv_maintainer, int32_t max_level, - const LookupStrategy& lookup_strategy, + const LookupStrategy& lookup_strategy, bool should_produce_changelog, + const std::shared_ptr::Factory>& processor_factory, const std::shared_ptr& path_factory_cache, const std::shared_ptr& cancellation_controller, const std::shared_ptr& remote_lookup_file_manager) const { - auto merge_engine = options_.GetMergeEngine(); PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, table_schema_->TrimmedPrimaryKeys()); - if (lookup_strategy.produce_changelog || merge_engine != MergeEngine::DEDUPLICATE || - !options_.GetSequenceField().empty()) { - auto processor_factory = std::make_shared(schema_); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr> lookup_levels, - CreateLookupLevelsInternal( - options_, schema_manager_, io_manager_, cache_manager_, file_store_path_factory_, - table_schema_, partition, bucket, levels, processor_factory, dv_maintainer, - lookup_file_cache_, remote_lookup_file_manager, pool_)); - auto merge_function_wrapper_factory = - [data_schema = schema_, options = options_, trimmed_primary_keys, - lookup_levels_ptr = lookup_levels.get(), lookup_strategy, - dv_maintainer_ptr = dv_maintainer, pool = pool_, - user_defined_seq_comparator = user_defined_seq_comparator_]( - int32_t output_level) -> Result>> { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merge_func, - PrimaryKeyTableUtils::CreateMergeFunction( - data_schema, trimmed_primary_keys, options, pool)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr> merge_function_wrapper, - LookupMergeTreeCompactRewriter:: - CreateLookupMergeFunctionWrapper( - std::make_unique(std::move(merge_func)), output_level, - dv_maintainer_ptr, lookup_strategy, user_defined_seq_comparator, - lookup_levels_ptr)); - return merge_function_wrapper; - }; - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr> rewriter, - LookupMergeTreeCompactRewriter::Create( - max_level, std::move(lookup_levels), dv_maintainer, - std::move(merge_function_wrapper_factory), bucket, partition, table_schema_, - path_factory_cache, options_, cancellation_controller, remote_lookup_file_manager, - pool_)); - return std::shared_ptr(std::move(rewriter)); - } - auto processor_factory = std::make_shared(); PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr> lookup_levels, - CreateLookupLevelsInternal( - options_, schema_manager_, io_manager_, cache_manager_, file_store_path_factory_, - table_schema_, partition, bucket, levels, processor_factory, dv_maintainer, - lookup_file_cache_, remote_lookup_file_manager, pool_)); + std::unique_ptr> lookup_levels, + CreateLookupLevelsInternal(options_, schema_manager_, io_manager_, cache_manager_, + file_store_path_factory_, table_schema_, partition, bucket, + levels, processor_factory, dv_maintainer, lookup_file_cache_, + remote_lookup_file_manager, pool_)); auto merge_function_wrapper_factory = [data_schema = schema_, options = options_, trimmed_primary_keys, - lookup_levels_ptr = lookup_levels.get(), lookup_strategy, - dv_maintainer_ptr = dv_maintainer, pool = pool_, - user_defined_seq_comparator = user_defined_seq_comparator_]( - int32_t output_level) -> Result>> { + lookup_levels_ptr = lookup_levels.get(), lookup_strategy, should_produce_changelog, + dv_maintainer_ptr = dv_maintainer, + user_defined_seq_comparator = user_defined_seq_comparator_, + pool = pool_](int32_t output_level) + -> Result>> { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merge_func, PrimaryKeyTableUtils::CreateMergeFunction( data_schema, trimmed_primary_keys, options, pool)); PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr> merge_function_wrapper, - LookupMergeTreeCompactRewriter::CreateLookupMergeFunctionWrapper( - std::make_unique(std::move(merge_func)), output_level, - dv_maintainer_ptr, lookup_strategy, user_defined_seq_comparator, - lookup_levels_ptr)); - return merge_function_wrapper; + std::unique_ptr value_serializer, + CreateChangelogValueSerializer(data_schema, should_produce_changelog, pool)); + PAIMON_ASSIGN_OR_RAISE( + FieldsComparator::FieldComparatorFunc value_equalizer, + CreateChangelogValueEqualizer(data_schema, options, should_produce_changelog)); + return LookupMergeTreeCompactRewriter::CreateLookupMergeFunctionWrapper( + std::make_unique(std::move(merge_func)), output_level, + dv_maintainer_ptr, lookup_strategy, should_produce_changelog, + user_defined_seq_comparator, std::move(value_serializer), std::move(value_equalizer), + lookup_levels_ptr); }; - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr> rewriter, - LookupMergeTreeCompactRewriter::Create( - max_level, std::move(lookup_levels), dv_maintainer, - std::move(merge_function_wrapper_factory), bucket, partition, - table_schema_, path_factory_cache, options_, cancellation_controller, - remote_lookup_file_manager, pool_)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr> rewriter, + LookupMergeTreeCompactRewriter::Create( + max_level, std::move(lookup_levels), dv_maintainer, + std::move(merge_function_wrapper_factory), bucket, partition, table_schema_, + path_factory_cache, options_, should_produce_changelog, cancellation_controller, + remote_lookup_file_manager, pool_)); return std::shared_ptr(std::move(rewriter)); } Result> -MergeTreeCompactManagerFactory::CreateLookupRewriterWithoutDeletionVector( +MergeTreeCompactManagerFactory::CreateLookupRewriterWithDeletionVector( const BinaryRow& partition, int32_t bucket, const std::shared_ptr& levels, const std::shared_ptr& dv_maintainer, int32_t max_level, - const LookupStrategy& lookup_strategy, + const LookupStrategy& lookup_strategy, bool should_produce_changelog, const std::shared_ptr& path_factory_cache, const std::shared_ptr& cancellation_controller, const std::shared_ptr& remote_lookup_file_manager) const { - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, - table_schema_->TrimmedPrimaryKeys()); - auto processor_factory = std::make_shared(schema_); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr> lookup_levels, - CreateLookupLevelsInternal( - options_, schema_manager_, io_manager_, cache_manager_, file_store_path_factory_, - table_schema_, partition, bucket, levels, processor_factory, dv_maintainer, - lookup_file_cache_, remote_lookup_file_manager, pool_)); - auto merge_function_wrapper_factory = - [data_schema = schema_, options = options_, trimmed_primary_keys, - lookup_levels_ptr = lookup_levels.get(), lookup_strategy, - dv_maintainer_ptr = dv_maintainer, pool = pool_, - user_defined_seq_comparator = user_defined_seq_comparator_]( - int32_t output_level) -> Result>> { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merge_func, - PrimaryKeyTableUtils::CreateMergeFunction( - data_schema, trimmed_primary_keys, options, pool)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr> merge_function_wrapper, - LookupMergeTreeCompactRewriter::CreateLookupMergeFunctionWrapper( - std::make_unique(std::move(merge_func)), output_level, - dv_maintainer_ptr, lookup_strategy, user_defined_seq_comparator, - lookup_levels_ptr)); - return merge_function_wrapper; - }; + auto merge_engine = options_.GetMergeEngine(); + if (lookup_strategy.produce_changelog || merge_engine != MergeEngine::DEDUPLICATE || + !options_.GetSequenceField().empty()) { + std::shared_ptr::Factory> processor_factory = + std::make_shared(schema_); + return CreateLookupRewriterInternal( + partition, bucket, levels, dv_maintainer, max_level, lookup_strategy, + should_produce_changelog, processor_factory, path_factory_cache, + cancellation_controller, remote_lookup_file_manager); + } + std::shared_ptr::Factory> processor_factory = + std::make_shared(); + return CreateLookupRewriterInternal( + partition, bucket, levels, dv_maintainer, max_level, lookup_strategy, + should_produce_changelog, processor_factory, path_factory_cache, cancellation_controller, + remote_lookup_file_manager); +} - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr> rewriter, - LookupMergeTreeCompactRewriter::Create( - max_level, std::move(lookup_levels), dv_maintainer, - std::move(merge_function_wrapper_factory), bucket, partition, - table_schema_, path_factory_cache, options_, cancellation_controller, - remote_lookup_file_manager, pool_)); - return std::shared_ptr(std::move(rewriter)); +Result> +MergeTreeCompactManagerFactory::CreateLookupRewriterWithoutDeletionVector( + const BinaryRow& partition, int32_t bucket, const std::shared_ptr& levels, + const std::shared_ptr& dv_maintainer, int32_t max_level, + const LookupStrategy& lookup_strategy, bool should_produce_changelog, + const std::shared_ptr& path_factory_cache, + const std::shared_ptr& cancellation_controller, + const std::shared_ptr& remote_lookup_file_manager) const { + std::shared_ptr::Factory> processor_factory = + std::make_shared(schema_); + return CreateLookupRewriterInternal( + partition, bucket, levels, dv_maintainer, max_level, lookup_strategy, + should_produce_changelog, processor_factory, path_factory_cache, cancellation_controller, + remote_lookup_file_manager); } Result> diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.h b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.h index 13a059620..2eaa6ffd4 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.h +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.h @@ -71,7 +71,8 @@ class MergeTreeCompactManagerFactory { const std::shared_ptr& io_manager, const std::shared_ptr& cache_manager, const std::shared_ptr& file_store_path_factory, - const std::string& root_path, const std::shared_ptr& pool) + const std::string& root_path, bool ignore_previous_files, + const std::shared_ptr& pool) : options_(options), pool_(pool), key_comparator_(key_comparator), @@ -83,7 +84,8 @@ class MergeTreeCompactManagerFactory { io_manager_(io_manager), cache_manager_(cache_manager), file_store_path_factory_(file_store_path_factory), - root_path_(root_path) {} + root_path_(root_path), + ignore_previous_files_(ignore_previous_files) {} std::shared_ptr CreateCompactStrategy() const; @@ -115,10 +117,20 @@ class MergeTreeCompactManagerFactory { const std::shared_ptr& path_factory_cache, const std::shared_ptr& cancellation_controller) const; + template + Result> CreateLookupRewriterInternal( + const BinaryRow& partition, int32_t bucket, const std::shared_ptr& levels, + const std::shared_ptr& dv_maintainer, int32_t max_level, + const LookupStrategy& lookup_strategy, bool should_produce_changelog, + const std::shared_ptr::Factory>& processor_factory, + const std::shared_ptr& path_factory_cache, + const std::shared_ptr& cancellation_controller, + const std::shared_ptr& remote_lookup_file_manager) const; + Result> CreateLookupRewriterWithDeletionVector( const BinaryRow& partition, int32_t bucket, const std::shared_ptr& levels, const std::shared_ptr& dv_maintainer, int32_t max_level, - const LookupStrategy& lookup_strategy, + const LookupStrategy& lookup_strategy, bool should_produce_changelog, const std::shared_ptr& path_factory_cache, const std::shared_ptr& cancellation_controller, const std::shared_ptr& remote_lookup_file_manager) const; @@ -126,7 +138,7 @@ class MergeTreeCompactManagerFactory { Result> CreateLookupRewriterWithoutDeletionVector( const BinaryRow& partition, int32_t bucket, const std::shared_ptr& levels, const std::shared_ptr& dv_maintainer, int32_t max_level, - const LookupStrategy& lookup_strategy, + const LookupStrategy& lookup_strategy, bool should_produce_changelog, const std::shared_ptr& path_factory_cache, const std::shared_ptr& cancellation_controller, const std::shared_ptr& remote_lookup_file_manager) const; @@ -146,6 +158,7 @@ class MergeTreeCompactManagerFactory { std::shared_ptr cache_manager_; std::shared_ptr file_store_path_factory_; std::string root_path_; + bool ignore_previous_files_; std::shared_ptr lookup_file_cache_; }; diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp index 91e3bb1cc..06a4ec197 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp @@ -88,6 +88,7 @@ class MergeTreeCompactManagerFactoryStrategyTest : public ::testing::Test { /*cache_manager=*/nullptr, /*file_store_path_factory=*/nullptr, /*root_path=*/"", + /*ignore_previous_files=*/false, /*pool=*/nullptr); } }; @@ -333,7 +334,8 @@ TEST_F(MergeTreeCompactManagerFactoryWriteTest, ASSERT_NOK_WITH_MSG(CreateSingleStringFileStoreWrite( {{"bucket", "1"}, {Options::CHANGELOG_PRODUCER, "full-compaction"}}, /*with_io_manager=*/false), - "C++ Paimon does not support changelog-producer yet"); + "C++ Paimon only supports 'none', 'input' and 'lookup' " + "changelog-producer now"); } TEST_F(MergeTreeCompactManagerFactoryWriteTest, @@ -378,13 +380,11 @@ TEST_F(MergeTreeCompactManagerFactoryWriteTest, } TEST_F(MergeTreeCompactManagerFactoryWriteTest, - TestCreateFileStoreWriteShouldFailWhenLookupChangelogConfigured) { - ASSERT_NOK_WITH_MSG( - CreateSingleStringFileStoreWrite({{"bucket", "1"}, - {Options::DELETION_VECTORS_ENABLED, "true"}, - {Options::CHANGELOG_PRODUCER, "lookup"}}, - /*with_io_manager=*/true), - "C++ Paimon does not support changelog-producer yet"); + TestCreateFileStoreWriteShouldSucceedWhenLookupChangelogConfigured) { + ASSERT_OK(CreateSingleStringFileStoreWrite({{"bucket", "1"}, + {Options::DELETION_VECTORS_ENABLED, "true"}, + {Options::CHANGELOG_PRODUCER, "lookup"}}, + /*with_io_manager=*/true)); } } // namespace paimon::test diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp index fd7c7cbd2..37c1cfa04 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp @@ -23,14 +23,12 @@ #include "arrow/c/bridge.h" #include "arrow/c/helpers.h" -#include "paimon/common/data/shredding/shredding_write_plan_factories.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/utils/scope_guard.h" -#include "paimon/core/io/key_value_data_file_writer_factory.h" +#include "paimon/core/io/key_value_data_file_writer_factories.h" #include "paimon/core/io/key_value_meta_projection_consumer.h" #include "paimon/core/io/key_value_record_reader.h" #include "paimon/core/io/row_to_arrow_array_converter.h" -#include "paimon/core/io/shredding_key_value_data_file_writer_factory.h" #include "paimon/core/manifest/file_source.h" #include "paimon/core/operation/internal_read_context.h" #include "paimon/format/file_format.h" @@ -133,20 +131,31 @@ MergeTreeCompactRewriter::CreateRollingRowWriter(int32_t level) { auto format = options_.GetWriteFileFormat(level); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_file_path_factory, CreateDataFilePathFactory(format->Identifier())); - std::shared_ptr>> factory; PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr plan_factory, - ShreddingWritePlanFactories::SelectActive(options_, write_schema_, pool_)); - if (plan_factory != nullptr) { - factory = std::make_shared( + std::shared_ptr factory, + KeyValueDataFileWriterFactories::Create( options_, schema_id_, write_schema_, level, FileSource::Compact(), trimmed_primary_keys_, data_file_path_factory, /*create_stats_extractor=*/true, - plan_factory, pool_); - } else { - factory = std::make_shared( - options_, schema_id_, write_schema_, level, FileSource::Compact(), - trimmed_primary_keys_, data_file_path_factory, /*create_stats_extractor=*/true, pool_); + /*is_changelog=*/false, pool_)); + return std::make_unique( + options_.GetTargetFileSize(/*has_primary_key=*/true), + /*target_file_row_num=*/std::numeric_limits::max(), factory); +} + +Result> +MergeTreeCompactRewriter::CreateRollingChangelogWriter(int32_t level) { + std::shared_ptr format = options_.GetChangelogFileFormat(); + if (!format) { + format = options_.GetWriteFileFormat(level); } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_file_path_factory, + CreateDataFilePathFactory(format->Identifier())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr factory, + KeyValueDataFileWriterFactories::Create( + options_, schema_id_, write_schema_, level, FileSource::Append(), trimmed_primary_keys_, + data_file_path_factory, /*create_stats_extractor=*/true, + /*is_changelog=*/true, pool_)); return std::make_unique( options_.GetTargetFileSize(/*has_primary_key=*/true), /*target_file_row_num=*/std::numeric_limits::max(), factory); @@ -178,6 +187,19 @@ Result> MergeTreeCompactRewriter::CreateDat return path_factory->CreateDataFilePathFactory(partition_, bucket_); } +Result> +MergeTreeCompactRewriter::CreateRawSortMergeReaderForSection( + const std::vector& section) { + if (!merge_file_split_read_) { + return Status::Invalid( + "merge_file_split_read in MergeTreeCompactRewriter cannot be nullptr"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_file_path_factory, + CreateDataFilePathFactory(options_.GetFileFormat()->Identifier())); + return merge_file_split_read_->CreateRawSortMergeReaderForSection( + section, partition_, dv_factory_, /*predicate=*/nullptr, data_file_path_factory); +} + Status MergeTreeCompactRewriter::MergeReadAndWrite( int32_t output_level, bool drop_delete, const std::vector& section, const MergeTreeCompactRewriter::KeyValueConsumerCreator& create_consumer, @@ -222,11 +244,12 @@ Status MergeTreeCompactRewriter::MergeReadAndWrite( return Status::OK(); } - // consumer batch size is WriteBatchSize + std::unique_ptr producer = + std::make_unique(std::move(sort_merge_reader), + options_.GetWriteBatchSize()); auto async_key_value_producer_consumer = std::make_shared>( - std::move(sort_merge_reader), create_consumer, options_.GetWriteBatchSize(), - /*projection_thread_num=*/1, pool_); + std::move(producer), create_consumer, /*projection_thread_num=*/1); reader_holders.push_back(async_key_value_producer_consumer); // read KeyValueBatch from SortMergeReader and write to RollingWriter while (true) { diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.h b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.h index c9c624704..c15e16fc5 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.h +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.h @@ -92,6 +92,8 @@ class MergeTreeCompactRewriter : public CompactRewriter { Result> CreateRollingRowWriter(int32_t level); + Result> CreateRollingChangelogWriter(int32_t level); + Result GenerateKeyValueConsumer() const; Status MergeReadAndWrite(int32_t output_level, bool drop_delete, @@ -100,6 +102,13 @@ class MergeTreeCompactRewriter : public CompactRewriter { KeyValueRollingFileWriter* rolling_writer, std::vector>* reader_holders_ptr); + Result> CreateRawSortMergeReaderForSection( + const std::vector& section); + + bool IsCancelled() const { + return cancellation_controller_->IsCancelled(); + } + protected: CoreOptions options_; std::unique_ptr merge_file_split_read_; @@ -108,7 +117,6 @@ class MergeTreeCompactRewriter : public CompactRewriter { Result> CreateDataFilePathFactory( const std::string& format); - private: std::shared_ptr pool_; BinaryRow partition_; int32_t bucket_; diff --git a/src/paimon/core/mergetree/external_sort_buffer.cpp b/src/paimon/core/mergetree/external_sort_buffer.cpp index 9bfeec899..718a2de62 100644 --- a/src/paimon/core/mergetree/external_sort_buffer.cpp +++ b/src/paimon/core/mergetree/external_sort_buffer.cpp @@ -222,10 +222,11 @@ Result ExternalSortBuffer::SpillToDisk( -> Result>> { return KeyValueMetaProjectionConsumer::Create(target_schema, pool); }; + std::unique_ptr producer = + std::make_unique(std::move(sorted_reader), write_batch_size); auto async_key_value_producer_consumer = std::make_unique>( - std::move(sorted_reader), create_consumer, write_batch_size, - /*projection_thread_num=*/1, pool_); + std::move(producer), create_consumer, /*projection_thread_num=*/1); auto close_guard = ScopeGuard([&]() { async_key_value_producer_consumer->Close(); }); while (true) { diff --git a/src/paimon/core/mergetree/merge_tree_writer.cpp b/src/paimon/core/mergetree/merge_tree_writer.cpp index 3b6806c73..bb7a01054 100644 --- a/src/paimon/core/mergetree/merge_tree_writer.cpp +++ b/src/paimon/core/mergetree/merge_tree_writer.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -27,7 +28,6 @@ #include "arrow/api.h" #include "arrow/c/abi.h" #include "arrow/c/helpers.h" -#include "paimon/common/data/shredding/shredding_write_plan_factories.h" #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -37,13 +37,13 @@ #include "paimon/core/io/compact_increment.h" #include "paimon/core/io/data_file_path_factory.h" #include "paimon/core/io/data_increment.h" -#include "paimon/core/io/key_value_data_file_writer_factory.h" +#include "paimon/core/io/key_value_data_file_writer_factories.h" #include "paimon/core/io/key_value_meta_projection_consumer.h" #include "paimon/core/io/key_value_record_reader.h" #include "paimon/core/io/row_to_arrow_array_converter.h" -#include "paimon/core/io/shredding_key_value_data_file_writer_factory.h" #include "paimon/core/manifest/file_source.h" #include "paimon/core/mergetree/compact/sort_merge_reader_with_loser_tree.h" +#include "paimon/core/mergetree/compact/sort_merge_reader_with_min_heap.h" #include "paimon/core/mergetree/write_buffer.h" #include "paimon/core/utils/commit_increment.h" @@ -104,8 +104,13 @@ Status MergeTreeWriter::DoClose() { // delete temporary files std::vector> delete_files; - delete_files.reserve(new_files_.size() + compact_after_.size()); + delete_files.reserve(new_files_.size() + new_changelog_files_.size() + compact_after_.size() + + compact_changelog_files_.size()); delete_files.insert(delete_files.end(), new_files_.begin(), new_files_.end()); + delete_files.insert(delete_files.end(), new_changelog_files_.begin(), + new_changelog_files_.end()); + delete_files.insert(delete_files.end(), compact_changelog_files_.begin(), + compact_changelog_files_.end()); for (const auto& file : compact_after_) { // Upgrade file is required by previous snapshot, so we should ensure that this file is // not the output of upgraded. @@ -125,9 +130,11 @@ Status MergeTreeWriter::DoClose() { write_buffer_->Clear(); new_files_.clear(); + new_changelog_files_.clear(); deleted_files_.clear(); compact_before_.clear(); compact_after_.clear(); + compact_changelog_files_.clear(); if (compact_deletion_file_) { compact_deletion_file_->Clean(); @@ -210,7 +217,9 @@ Status MergeTreeWriter::UpdateCompactResult(const std::shared_ptr compact_after_.insert(compact_after_.end(), compact_result->After().begin(), compact_result->After().end()); - // TODO(yonghao.fyh): support compact changelog + compact_changelog_files_.insert(compact_changelog_files_.end(), + compact_result->Changelog().begin(), + compact_result->Changelog().end()); return UpdateCompactDeletionFile(compact_result->DeletionFile()); } @@ -256,23 +265,62 @@ Status MergeTreeWriter::FlushWriteBuffer(bool wait_for_latest_compaction, wait_for_latest_compaction = true; } auto cleanup_guard = ScopeGuard([&]() { write_buffer_->Clear(); }); - // 1. flush write buffer to get sorted readers + + auto create_consumer = [target_schema = write_schema_, pool = pool_]() + -> Result>> { + return KeyValueMetaProjectionConsumer::Create(target_schema, pool); + }; + + std::unique_ptr> + async_changelog_producer_consumer; + std::unique_ptr>> + changelog_writer; + std::vector> flushed_changelog_files; + ScopeGuard changelog_write_guard([&]() -> void { + if (changelog_writer) { + changelog_writer->Abort(); + } + if (async_changelog_producer_consumer) { + async_changelog_producer_consumer->Close(); + } + }); + if (options_.GetChangelogProducer() == ChangelogProducer::INPUT) { + PAIMON_ASSIGN_OR_RAISE(std::vector> raw_readers, + write_buffer_->CreateRawReaders()); + auto raw_sort_merge_reader = std::make_unique( + std::move(raw_readers), key_comparator_, user_defined_seq_comparator_, + /*merge_function_wrapper=*/nullptr); + std::unique_ptr producer = + std::make_unique(std::move(raw_sort_merge_reader), + options_.GetWriteBatchSize()); + async_changelog_producer_consumer = + std::make_unique>( + std::move(producer), create_consumer, /*projection_thread_num=*/1); + PAIMON_ASSIGN_OR_RAISE(changelog_writer, CreateRollingChangelogWriter()); + while (true) { + PAIMON_ASSIGN_OR_RAISE(KeyValueBatch key_value_batch, + async_changelog_producer_consumer->NextBatch()); + if (key_value_batch.batch == nullptr) { + break; + } + PAIMON_RETURN_NOT_OK(changelog_writer->Write(std::move(key_value_batch))); + } + PAIMON_RETURN_NOT_OK(changelog_writer->Close()); + PAIMON_ASSIGN_OR_RAISE(flushed_changelog_files, changelog_writer->GetResult()); + } + + // Flush write buffer to get sorted and merged data readers. PAIMON_ASSIGN_OR_RAISE(std::vector> readers, write_buffer_->CreateReaders()); - // 2. prepare loser tree sort merge reader auto sort_merge_reader = std::make_unique( std::move(readers), key_comparator_, user_defined_seq_comparator_, merge_function_wrapper_); - // 3. project key value to arrow array - auto create_consumer = [target_schema = write_schema_, pool = pool_]() - -> Result>> { - return KeyValueMetaProjectionConsumer::Create(target_schema, pool); - }; - // consumer batch size is WriteBatchSize + std::unique_ptr producer = + std::make_unique(std::move(sort_merge_reader), + options_.GetWriteBatchSize()); auto async_key_value_producer_consumer = std::make_unique>( - std::move(sort_merge_reader), create_consumer, options_.GetWriteBatchSize(), - /*projection_thread_num=*/1, pool_); + std::move(producer), create_consumer, /*projection_thread_num=*/1); std::unique_ptr>> rolling_writer; PAIMON_ASSIGN_OR_RAISE(rolling_writer, CreateRollingRowWriter()); @@ -292,13 +340,24 @@ Status MergeTreeWriter::FlushWriteBuffer(bool wait_for_latest_compaction, PAIMON_ASSIGN_OR_RAISE(std::vector> flushed_files, rolling_writer->GetResult()); async_key_value_producer_consumer->Close(); + if (async_changelog_producer_consumer) { + async_changelog_producer_consumer->Close(); + } + + new_changelog_files_.insert(new_changelog_files_.end(), flushed_changelog_files.begin(), + flushed_changelog_files.end()); + new_files_.insert(new_files_.end(), flushed_files.begin(), flushed_files.end()); + write_guard.Release(); + changelog_write_guard.Release(); for (const auto& flushed_file : flushed_files) { - new_files_.emplace_back(flushed_file); PAIMON_RETURN_NOT_OK(compact_manager_->AddNewFile(flushed_file)); } metrics_->Merge(rolling_writer->GetMetrics()); + if (changelog_writer) { + metrics_->Merge(changelog_writer->GetMetrics()); + } } PAIMON_RETURN_NOT_OK(TrySyncLatestCompaction(wait_for_latest_compaction)); PAIMON_RETURN_NOT_OK(compact_manager_->TriggerCompaction(forced_full_compaction)); @@ -306,14 +365,18 @@ Status MergeTreeWriter::FlushWriteBuffer(bool wait_for_latest_compaction, } Result MergeTreeWriter::DrainIncrement() { - DataIncrement data_increment(std::move(new_files_), std::move(deleted_files_), {}); - CompactIncrement compact_increment(std::move(compact_before_), std::move(compact_after_), {}); + DataIncrement data_increment(std::move(new_files_), std::move(deleted_files_), + std::move(new_changelog_files_)); + CompactIncrement compact_increment(std::move(compact_before_), std::move(compact_after_), + std::move(compact_changelog_files_)); auto drain_deletion_file = compact_deletion_file_; new_files_.clear(); + new_changelog_files_.clear(); deleted_files_.clear(); compact_before_.clear(); compact_after_.clear(); + compact_changelog_files_.clear(); compact_deletion_file_ = nullptr; return CommitIncrement(data_increment, compact_increment, drain_deletion_file); @@ -321,23 +384,28 @@ Result MergeTreeWriter::DrainIncrement() { Result>>> MergeTreeWriter::CreateRollingRowWriter() const { - std::shared_ptr>> factory; PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr plan_factory, - ShreddingWritePlanFactories::SelectActive(options_, write_schema_, pool_)); - if (plan_factory != nullptr) { - factory = std::make_shared( - options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(), - trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/true, plan_factory, - pool_); - } else { - factory = std::make_shared( - options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(), - trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/true, pool_); - } + std::shared_ptr factory, + KeyValueDataFileWriterFactories::Create(options_, schema_id_, write_schema_, /*level=*/0, + FileSource::Append(), trimmed_primary_keys_, + path_factory_, /*create_stats_extractor=*/true, + /*is_changelog=*/false, pool_)); return std::make_unique>>( options_.GetTargetFileSize(/*has_primary_key=*/true), options_.GetTargetFileRowNum(), factory); } +Result>>> +MergeTreeWriter::CreateRollingChangelogWriter() const { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr factory, + KeyValueDataFileWriterFactories::Create(options_, schema_id_, write_schema_, /*level=*/0, + FileSource::Append(), trimmed_primary_keys_, + path_factory_, /*create_stats_extractor=*/true, + /*is_changelog=*/true, pool_)); + return std::make_unique>>( + options_.GetTargetFileSize(/*has_primary_key=*/true), + /*target_file_row_num=*/std::numeric_limits::max(), factory); +} + } // namespace paimon diff --git a/src/paimon/core/mergetree/merge_tree_writer.h b/src/paimon/core/mergetree/merge_tree_writer.h index febce2afb..10f35fbbe 100644 --- a/src/paimon/core/mergetree/merge_tree_writer.h +++ b/src/paimon/core/mergetree/merge_tree_writer.h @@ -100,6 +100,9 @@ class MergeTreeWriter : public BatchWriter { Result>>> CreateRollingRowWriter() const; + Result>>> + CreateRollingChangelogWriter() const; + Status TrySyncLatestCompaction(bool blocking); Status UpdateCompactResult(const std::shared_ptr& compact_result); Status UpdateCompactDeletionFile(const std::shared_ptr& new_deletion_file); @@ -134,9 +137,11 @@ class MergeTreeWriter : public BatchWriter { std::shared_ptr metrics_; std::vector> new_files_; + std::vector> new_changelog_files_; std::vector> deleted_files_; std::vector> compact_before_; std::vector> compact_after_; + std::vector> compact_changelog_files_; std::shared_ptr compact_deletion_file_; }; diff --git a/src/paimon/core/mergetree/merge_tree_writer_test.cpp b/src/paimon/core/mergetree/merge_tree_writer_test.cpp index c5a114f36..b3f53f487 100644 --- a/src/paimon/core/mergetree/merge_tree_writer_test.cpp +++ b/src/paimon/core/mergetree/merge_tree_writer_test.cpp @@ -37,6 +37,7 @@ #include "paimon/common/types/data_field.h" #include "paimon/common/utils/fields_comparator.h" #include "paimon/common/utils/scope_guard.h" +#include "paimon/common/utils/string_utils.h" #include "paimon/core/compact/noop_compact_manager.h" #include "paimon/core/disk/io_manager.h" #include "paimon/core/io/compact_increment.h" @@ -144,11 +145,13 @@ class MergeTreeWriterTest : public ::testing::TestWithParam { } void CheckFileContent(const std::string& data_file_name, - const std::shared_ptr& expected_array) const { + const std::shared_ptr& expected_array, + const std::string& file_format_name = "orc") const { ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, file_system_->Open(data_file_name)); ASSERT_TRUE(input_stream); - ASSERT_OK_AND_ASSIGN(auto file_format, FileFormatFactory::Get("orc", /*options=*/{})); + ASSERT_OK_AND_ASSIGN(auto file_format, + FileFormatFactory::Get(file_format_name, /*options=*/{})); ASSERT_OK_AND_ASSIGN(auto reader_builder, file_format->CreateReaderBuilder(/*batch_size=*/10)); ASSERT_OK_AND_ASSIGN(auto orc_batch_reader, reader_builder->Build(input_stream)); @@ -160,11 +163,13 @@ class MergeTreeWriterTest : public ::testing::TestWithParam { void CheckShreddingFileSchema(const std::string& data_file_name, const std::shared_ptr& expected_physical_schema, int32_t field_index, - const MapSharedShreddingFieldMeta& expected_meta) const { + const MapSharedShreddingFieldMeta& expected_meta, + const std::string& file_format_name = "orc") const { ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, file_system_->Open(data_file_name)); ASSERT_TRUE(input_stream); - ASSERT_OK_AND_ASSIGN(auto file_format, FileFormatFactory::Get("orc", /*options=*/{})); + ASSERT_OK_AND_ASSIGN(auto file_format, + FileFormatFactory::Get(file_format_name, /*options=*/{})); ASSERT_OK_AND_ASSIGN(auto reader_builder, file_format->CreateReaderBuilder(/*batch_size=*/10)); ASSERT_OK_AND_ASSIGN(auto orc_batch_reader, reader_builder->Build(input_stream)); @@ -296,6 +301,182 @@ TEST_P(MergeTreeWriterTest, TestSimple) { ASSERT_EQ(expected_data_increment, commit_increment.GetNewFilesIncrement()); } +TEST_P(MergeTreeWriterTest, TestInputChangelog) { + ASSERT_OK_AND_ASSIGN(CoreOptions options, + CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}, + {Options::CHANGELOG_PRODUCER, "input"}, + {Options::CHANGELOG_FILE_PREFIX, "changes-"}, + {Options::CHANGELOG_FILE_FORMAT, "parquet"}})); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = std::make_shared(); + ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr)); + std::string uuid = path_factory->uuid_; + + ASSERT_OK_AND_ASSIGN(auto merge_writer, + CreateMergeWriter(/*last_sequence_number=*/-1, dir->Str(), path_factory, + /*schema_id=*/1, options)); + + std::shared_ptr array = + arrow::ipc::internal::json::ArrayFromJSON(value_type_, R"([ + ["Bob", 20, 0, 20.0], + ["Alice", 10, 0, 10.0], + ["Alice", 11, 0, 11.0], + ["Bob", 21, 0, 21.0] + ])") + .ValueOrDie(); + WriteBatch(array, + {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::UPDATE_BEFORE, + RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::DELETE}, + merge_writer.get()); + if (GetParam()) { + ASSERT_OK(merge_writer->FlushMemory()); + } + + ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment, + merge_writer->PrepareCommit(/*wait_compaction=*/false)); + ASSERT_OK(merge_writer->Close()); + + const DataIncrement& data_increment = commit_increment.GetNewFilesIncrement(); + ASSERT_EQ(1, data_increment.NewFiles().size()); + ASSERT_EQ(1, data_increment.ChangelogFiles().size()); + ASSERT_EQ("data-" + uuid + "-1.orc", data_increment.NewFiles()[0]->file_name); + ASSERT_EQ("changes-" + uuid + "-0.parquet", data_increment.ChangelogFiles()[0]->file_name); + ASSERT_EQ(2, data_increment.NewFiles()[0]->row_count); + ASSERT_EQ(4, data_increment.ChangelogFiles()[0]->row_count); + + std::shared_ptr expected_data; + auto data_status = arrow::ipc::internal::json::ChunkedArrayFromJSON(write_type_, {R"([ + [2, 2, "Alice", 11, 0, 11.0], + [3, 3, "Bob", 21, 0, 21.0] + ])"}, + &expected_data); + ASSERT_TRUE(data_status.ok()); + CheckFileContent(dir->Str() + "/" + data_increment.NewFiles()[0]->file_name, expected_data); + + std::shared_ptr expected_changelog; + auto changelog_status = arrow::ipc::internal::json::ChunkedArrayFromJSON(write_type_, {R"([ + [1, 1, "Alice", 10, 0, 10.0], + [2, 2, "Alice", 11, 0, 11.0], + [0, 0, "Bob", 20, 0, 20.0], + [3, 3, "Bob", 21, 0, 21.0] + ])"}, + &expected_changelog); + ASSERT_TRUE(changelog_status.ok()); + CheckFileContent(dir->Str() + "/" + data_increment.ChangelogFiles()[0]->file_name, + expected_changelog, "parquet"); +} + +TEST_P(MergeTreeWriterTest, TestInputChangelogIgnoresTargetFileRowNum) { + ASSERT_OK_AND_ASSIGN(CoreOptions options, + CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}, + {Options::CHANGELOG_PRODUCER, "input"}, + {Options::TARGET_FILE_ROW_NUM, "1"}, + {Options::WRITE_BATCH_SIZE, "1"}})); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = std::make_shared(); + ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr)); + ASSERT_OK_AND_ASSIGN(auto merge_writer, + CreateMergeWriter(/*last_sequence_number=*/-1, dir->Str(), path_factory, + /*schema_id=*/1, options)); + + std::shared_ptr array = + arrow::ipc::internal::json::ArrayFromJSON(value_type_, R"([ + ["Bob", 20, 0, 20.0], + ["Alice", 10, 0, 10.0], + ["Alice", 11, 0, 11.0], + ["Bob", 21, 0, 21.0] + ])") + .ValueOrDie(); + WriteBatch(array, /*row_kinds=*/{}, merge_writer.get()); + if (GetParam()) { + ASSERT_OK(merge_writer->FlushMemory()); + } + + ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment, + merge_writer->PrepareCommit(/*wait_compaction=*/false)); + ASSERT_OK(merge_writer->Close()); + + const std::vector>& changelog_files = + commit_increment.GetNewFilesIncrement().ChangelogFiles(); + ASSERT_EQ(1, changelog_files.size()); + ASSERT_EQ(4, changelog_files[0]->row_count); +} + +TEST_P(MergeTreeWriterTest, TestInputChangelogWithSharedShredding) { + ASSERT_OK_AND_ASSIGN(CoreOptions options, + CoreOptions::FromMap({ + {Options::FILE_FORMAT, "orc"}, + {Options::CHANGELOG_PRODUCER, "input"}, + {Options::CHANGELOG_FILE_PREFIX, "changes-"}, + {Options::CHANGELOG_FILE_FORMAT, "parquet"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "3"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, + {Options::WRITE_ONLY, "true"}, + })); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = std::make_shared(); + ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr)); + + std::vector value_fields = { + DataField(0, arrow::field("id", arrow::int32())), + DataField(1, arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64()))), + }; + auto value_schema = DataField::ConvertDataFieldsToArrowSchema(value_fields); + auto value_type = DataField::ConvertDataFieldsToArrowStructType(value_fields); + auto write_schema = SpecialFields::CompleteSequenceAndValueKindField(value_schema); + ASSERT_OK_AND_ASSIGN(std::shared_ptr key_comparator, + FieldsComparator::Create({value_fields[0]}, + /*is_ascending_order=*/true)); + ASSERT_OK_AND_ASSIGN( + auto merge_writer, + MergeTreeWriter::Create( + /*last_sequence_number=*/-1, /*trimmed_primary_keys=*/{"id"}, path_factory, + key_comparator, + /*user_defined_seq_comparator=*/nullptr, merge_function_wrapper_, /*schema_id=*/5, + value_schema, options, noop_compact_manager_, + GetParam() ? std::make_shared(dir->Str() + "/tmp", file_system_) : nullptr, + /*enable_multi_thread_spill=*/false, pool_)); + + auto array = arrow::ipc::internal::json::ArrayFromJSON(value_type, R"([ + [1, [["a", 10], ["b", 20]]], + [2, [["c", 30]]], + [1, [["a", 11], ["c", 31]]] + ])") + .ValueOrDie(); + WriteBatch(array, /*row_kinds=*/{}, merge_writer.get()); + if (GetParam()) { + ASSERT_OK(merge_writer->FlushMemory()); + } + + ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment, + merge_writer->PrepareCommit(/*wait_compaction=*/false)); + ASSERT_OK(merge_writer->Close()); + + const std::vector>& changelog_files = + commit_increment.GetNewFilesIncrement().ChangelogFiles(); + ASSERT_EQ(1, changelog_files.size()); + ASSERT_EQ(3, changelog_files[0]->row_count); + ASSERT_TRUE(StringUtils::EndsWith(changelog_files[0]->file_name, ".parquet")); + + std::map column_to_k = {{"tags", 3}}; + ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + write_schema, column_to_k)); + MapSharedShreddingFieldMeta expected_shredding_meta; + expected_shredding_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; + expected_shredding_meta.field_to_columns = {{0, {0}}, {1, {1}}, {2, {0, 1}}}; + expected_shredding_meta.num_columns = 3; + expected_shredding_meta.max_row_width = 2; + CheckShreddingFileSchema(path_factory->ToPath(changelog_files[0]->file_name), physical_schema, + /*field_index=*/3, expected_shredding_meta, "parquet"); +} + TEST_P(MergeTreeWriterTest, TestWriteMultiBatch) { ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}})); @@ -1195,6 +1376,31 @@ TEST_P(MergeTreeWriterTest, TestUpdateCompactResultDeleteIntermediateFile) { ASSERT_EQ(merge_writer->compact_after_, std::vector>({file_y})); } +TEST_P(MergeTreeWriterTest, TestUpdateCompactResultPropagatesChangelog) { + ASSERT_OK_AND_ASSIGN(CoreOptions options, + CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}})); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = std::make_shared(); + ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr)); + + auto fake_compact_manager = std::make_shared(); + ASSERT_OK_AND_ASSIGN( + auto merge_writer, + CreateMergeWriter(/*last_sequence_number=*/-1, dir->Str(), path_factory, /*schema_id=*/0, + options, /*user_defined_seq_comparator=*/nullptr, fake_compact_manager)); + + auto changelog = CreateMeta("changelog", /*level=*/0); + auto compact_result = std::make_shared( + std::vector>(), std::vector>(), + std::vector>({changelog})); + ASSERT_OK(merge_writer->UpdateCompactResult(compact_result)); + + ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment, merge_writer->DrainIncrement()); + ASSERT_EQ(commit_increment.GetCompactIncrement().ChangelogFiles(), + std::vector>({changelog})); +} + TEST_P(MergeTreeWriterTest, TestUpdateCompactResultWithFileInCompactAfter) { ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}})); diff --git a/src/paimon/core/mergetree/write_buffer.cpp b/src/paimon/core/mergetree/write_buffer.cpp index 549975a33..d6c09a1cb 100644 --- a/src/paimon/core/mergetree/write_buffer.cpp +++ b/src/paimon/core/mergetree/write_buffer.cpp @@ -82,6 +82,10 @@ Result>> WriteBuffer::CreateRe return merged_readers; } +Result>> WriteBuffer::CreateRawReaders() { + return sort_buffer_->CreateReaders(); +} + Result WriteBuffer::FlushMemory() { return sort_buffer_->FlushMemory(); } diff --git a/src/paimon/core/mergetree/write_buffer.h b/src/paimon/core/mergetree/write_buffer.h index f4f231000..1c71f7faf 100644 --- a/src/paimon/core/mergetree/write_buffer.h +++ b/src/paimon/core/mergetree/write_buffer.h @@ -72,6 +72,10 @@ class WriteBuffer { /// @return list of KeyValueRecordReaders built from buffered data Result>> CreateReaders(); + /// Create KeyValueRecordReaders containing the raw input records without merging duplicate + /// keys. The caller should invoke Clear() after consuming the readers. + Result>> CreateRawReaders(); + /// Try to spill current buffered data. Return false when the call completed normally but the /// caller should fall back to FlushWriteBuffer before buffering more data. Result FlushMemory(); diff --git a/src/paimon/core/operation/file_store_scan.cpp b/src/paimon/core/operation/file_store_scan.cpp index 467ba6b27..befe7eb97 100644 --- a/src/paimon/core/operation/file_store_scan.cpp +++ b/src/paimon/core/operation/file_store_scan.cpp @@ -275,6 +275,8 @@ Status FileStoreScan::ReadManifestsWithSnapshot(const Snapshot& snapshot, return manifest_list_->ReadDataManifests(snapshot, manifests); case ScanMode::DELTA: return manifest_list_->ReadDeltaManifests(snapshot, manifests); + case ScanMode::CHANGELOG: + return manifest_list_->ReadChangelogManifests(snapshot, manifests); default: return Status::NotImplemented("Unknown scan mode ", std::to_string(static_cast(scan_mode_))); diff --git a/src/paimon/core/operation/key_value_file_store_scan.cpp b/src/paimon/core/operation/key_value_file_store_scan.cpp index 54197f871..54af98a0c 100644 --- a/src/paimon/core/operation/key_value_file_store_scan.cpp +++ b/src/paimon/core/operation/key_value_file_store_scan.cpp @@ -177,8 +177,13 @@ Result KeyValueFileStoreScan::IsValueFilterEnabled() const { return value_filter_force_enabled_; case ScanMode::DELTA: return false; + case ScanMode::CHANGELOG: { + ChangelogProducer producer = core_options_.GetChangelogProducer(); + return producer == ChangelogProducer::LOOKUP || + producer == ChangelogProducer::FULL_COMPACTION; + } default: - return Status::NotImplemented("only support ALL and DELTA scan mode"); + return Status::NotImplemented("unknown scan mode"); } } diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp index 08c5ea0c3..09755984d 100644 --- a/src/paimon/core/operation/key_value_file_store_write.cpp +++ b/src/paimon/core/operation/key_value_file_store_write.cpp @@ -73,7 +73,7 @@ KeyValueFileStoreWrite::KeyValueFileStoreWrite( compact_manager_factory_(std::make_unique( options_, key_comparator_, user_defined_seq_comparator_, compaction_metrics_, table_schema_, schema_, schema_manager_, io_manager_, cache_manager_, - file_store_path_factory_, root_path_, pool_)), + file_store_path_factory_, root_path_, ignore_previous_files, pool_)), logger_(Logger::GetLogger("KeyValueFileStoreWrite")) {} Result> KeyValueFileStoreWrite::CreateFileStoreScan( diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp index b753ea431..2e517cdff 100644 --- a/src/paimon/core/operation/merge_file_split_read.cpp +++ b/src/paimon/core/operation/merge_file_split_read.cpp @@ -491,6 +491,23 @@ Result> MergeFileSplitRead::CreateSortMergeRead return sort_merge_reader; } +Result> MergeFileSplitRead::CreateRawSortMergeReaderForSection( + const std::vector& section, const BinaryRow& partition, + DeletionVector::Factory dv_factory, const std::shared_ptr& predicate, + const std::shared_ptr& data_file_path_factory) { + std::vector> record_readers; + record_readers.reserve(section.size()); + for (const auto& run : section) { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr run_reader, + CreateReaderForRun(partition, run, dv_factory, predicate, data_file_path_factory)); + record_readers.emplace_back(std::move(run_reader)); + } + return std::make_unique(std::move(record_readers), key_comparator_, + user_defined_seq_comparator_, + /*merge_function_wrapper=*/nullptr); +} + Result> MergeFileSplitRead::CreateReaderForRun( const BinaryRow& partition, const SortedRun& sorted_run, DeletionVector::Factory dv_factory, const std::shared_ptr& predicate, diff --git a/src/paimon/core/operation/merge_file_split_read.h b/src/paimon/core/operation/merge_file_split_read.h index 11dcd0b37..bda6e2401 100644 --- a/src/paimon/core/operation/merge_file_split_read.h +++ b/src/paimon/core/operation/merge_file_split_read.h @@ -104,6 +104,12 @@ class MergeFileSplitRead : public AbstractSplitRead { DeletionVector::Factory dv_factory, const std::shared_ptr& predicate, const std::shared_ptr& data_file_path_factory, bool drop_delete); + /// Creates a min-heap reader which only sorts records and preserves duplicate keys. + Result> CreateRawSortMergeReaderForSection( + const std::vector& section, const BinaryRow& partition, + DeletionVector::Factory dv_factory, const std::shared_ptr& predicate, + const std::shared_ptr& data_file_path_factory); + std::shared_ptr GetPathFactory() const { return path_factory_; } diff --git a/src/paimon/core/postpone/postpone_bucket_writer.cpp b/src/paimon/core/postpone/postpone_bucket_writer.cpp index 45bcfd364..71dfbc074 100644 --- a/src/paimon/core/postpone/postpone_bucket_writer.cpp +++ b/src/paimon/core/postpone/postpone_bucket_writer.cpp @@ -31,7 +31,6 @@ #include "arrow/c/helpers.h" #include "arrow/scalar.h" #include "fmt/format.h" -#include "paimon/common/data/shredding/shredding_write_plan_factories.h" #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/types/row_kind.h" @@ -42,8 +41,7 @@ #include "paimon/core/io/compact_increment.h" #include "paimon/core/io/data_file_path_factory.h" #include "paimon/core/io/data_increment.h" -#include "paimon/core/io/key_value_data_file_writer_factory.h" -#include "paimon/core/io/shredding_key_value_data_file_writer_factory.h" +#include "paimon/core/io/key_value_data_file_writer_factories.h" #include "paimon/core/manifest/file_source.h" #include "paimon/core/utils/commit_increment.h" #include "paimon/format/file_format.h" @@ -242,20 +240,12 @@ PostponeBucketWriter::PrepareMinMaxKey( Result>>> PostponeBucketWriter::CreateRollingRowWriter() const { - std::shared_ptr>> factory; PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr plan_factory, - ShreddingWritePlanFactories::SelectActive(options_, write_schema_, pool_)); - if (plan_factory != nullptr) { - factory = std::make_shared( - options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(), - trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/false, plan_factory, - pool_); - } else { - factory = std::make_shared( - options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(), - trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/false, pool_); - } + std::shared_ptr factory, + KeyValueDataFileWriterFactories::Create(options_, schema_id_, write_schema_, /*level=*/0, + FileSource::Append(), trimmed_primary_keys_, + path_factory_, /*create_stats_extractor=*/false, + /*is_changelog=*/false, pool_)); return std::make_unique>>( options_.GetTargetFileSize(/*has_primary_key=*/true), options_.GetTargetFileRowNum(), factory); diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index a6844d9e6..90f508b47 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -52,6 +52,7 @@ #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/bucket_mode.h" #include "paimon/defs.h" +#include "paimon/format/file_format.h" #include "paimon/result.h" namespace paimon { @@ -163,14 +164,8 @@ Status SchemaValidation::ValidateTableSchema(const TableSchema& schema) { PAIMON_RETURN_NOT_OK(ValidateFieldsPrefix(schema, options)); PAIMON_RETURN_NOT_OK(ValidateSequenceField(schema, options)); PAIMON_RETURN_NOT_OK(ValidateSequenceGroup(schema, options)); + PAIMON_RETURN_NOT_OK(ValidateChangelogProducer(schema, options)); - ChangelogProducer changelog_producer = options.GetChangelogProducer(); - if (schema.PrimaryKeys().empty() && changelog_producer != ChangelogProducer::NONE) { - return Status::Invalid( - fmt::format("Can not set {} on table without primary keys, please define primary keys.", - Options::CHANGELOG_PRODUCER)); - } - PAIMON_RETURN_NOT_OK(ValidateChangelogProducer(options)); PAIMON_RETURN_NOT_OK(Preconditions::CheckState( options.GetExpireConfig().GetSnapshotRetainMin() > 0, std::string(Options::SNAPSHOT_NUM_RETAINED_MIN) + " should be at least 1")); @@ -324,10 +319,41 @@ Status SchemaValidation::ValidateBucket(const TableSchema& schema, const CoreOpt return Status::OK(); } -Status SchemaValidation::ValidateChangelogProducer(const CoreOptions& options) { - return Preconditions::CheckState(options.GetChangelogProducer() == ChangelogProducer::NONE, - "C++ Paimon does not support changelog-producer yet. Please " - "keep changelog-producer as 'none'."); +Status SchemaValidation::ValidateChangelogProducer(const TableSchema& schema, + const CoreOptions& options) { + ChangelogProducer changelog_producer = options.GetChangelogProducer(); + if (schema.PrimaryKeys().empty() && changelog_producer != ChangelogProducer::NONE) { + return Status::Invalid( + fmt::format("Can not set {} on table without primary keys, please define primary keys.", + Options::CHANGELOG_PRODUCER)); + } + + bool row_deduplicate = options.ChangelogRowDeduplicate(); + const std::vector& ignore_fields = + options.GetChangelogRowDeduplicateIgnoreFields(); + PAIMON_RETURN_NOT_OK(Preconditions::CheckState( + ignore_fields.empty() || row_deduplicate, "'{}' is only valid when '{}' is true.", + Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE_IGNORE_FIELDS, + Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE)); + PAIMON_RETURN_NOT_OK(Preconditions::CheckState( + ObjectUtils::ContainsAll(schema.FieldNames(), ignore_fields), + "Fields {} configured in '{}' can not be found in table schema.", ignore_fields, + Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE_IGNORE_FIELDS)); + PAIMON_RETURN_NOT_OK(Preconditions::CheckState( + !row_deduplicate || changelog_producer == ChangelogProducer::LOOKUP || + changelog_producer == ChangelogProducer::FULL_COMPACTION, + "'{}' is only valid for 'lookup' or 'full-compaction' changelog producer.", + Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE)); + PAIMON_RETURN_NOT_OK(Preconditions::CheckState( + changelog_producer == ChangelogProducer::NONE || + changelog_producer == ChangelogProducer::INPUT || + changelog_producer == ChangelogProducer::LOOKUP, + "C++ Paimon only supports 'none', 'input' and 'lookup' changelog-producer now.")); + return Preconditions::CheckState( + options.GetMergeEngine() != MergeEngine::FIRST_ROW || + changelog_producer == ChangelogProducer::NONE || + changelog_producer == ChangelogProducer::LOOKUP, + "Only support 'none' and 'lookup' changelog-producer on FIRST_ROW merge engine"); } Status SchemaValidation::ValidateForDeletionVectors(const CoreOptions& options) { @@ -725,10 +751,20 @@ Status SchemaValidation::ValidateMapStorageLayout(const TableSchema& schema, options.GetFileFormat()->Identifier())); PAIMON_RETURN_NOT_OK(ValidatePerLevelOption(options_map, Options::FILE_FORMAT_PER_LEVEL, ValidateSharedShreddingFileFormat)); + std::shared_ptr changelog_format = options.GetChangelogFileFormat(); + if (changelog_format) { + PAIMON_RETURN_NOT_OK(ValidateSharedShreddingFileFormat(Options::CHANGELOG_FILE_FORMAT, + changelog_format->Identifier())); + } PAIMON_RETURN_NOT_OK(ValidateSharedShreddingCompression(Options::FILE_COMPRESSION, options.GetFileCompression())); PAIMON_RETURN_NOT_OK(ValidatePerLevelOption(options_map, Options::FILE_COMPRESSION_PER_LEVEL, ValidateSharedShreddingCompression)); + std::optional changelog_compression = options.GetChangelogFileCompression(); + if (changelog_compression) { + PAIMON_RETURN_NOT_OK(ValidateSharedShreddingCompression(Options::CHANGELOG_FILE_COMPRESSION, + changelog_compression.value())); + } return Status::OK(); } diff --git a/src/paimon/core/schema/schema_validation.h b/src/paimon/core/schema/schema_validation.h index d331bf10f..388ab42e2 100644 --- a/src/paimon/core/schema/schema_validation.h +++ b/src/paimon/core/schema/schema_validation.h @@ -66,7 +66,7 @@ class SchemaValidation { static Status ValidateFieldsPrefix(const TableSchema& schema, const CoreOptions& options); static Status ValidateSequenceField(const TableSchema& schema, const CoreOptions& options); static Status ValidateSequenceGroup(const TableSchema& schema, const CoreOptions& options); - static Status ValidateChangelogProducer(const CoreOptions& options); + static Status ValidateChangelogProducer(const TableSchema& schema, const CoreOptions& options); static Status ValidateForDeletionVectors(const CoreOptions& options); static Status ValidateRowTracking(const TableSchema& table_schema, const CoreOptions& options); diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp index 65ab7f7d1..53d0c3c8c 100644 --- a/src/paimon/core/schema/schema_validation_test.cpp +++ b/src/paimon/core/schema/schema_validation_test.cpp @@ -714,7 +714,18 @@ TEST(SchemaValidationTest, ValidateDeletionVector) { std::shared_ptr table_schema, TableSchema::Create(/*schema_id=*/0, schema, partition_keys, primary_keys, options)); ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), - "C++ Paimon does not support changelog-producer yet"); + "C++ Paimon only supports 'none', 'input' and 'lookup' " + "changelog-producer now"); + } + { + std::map options = {{Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "f0"}, + {Options::DELETION_VECTORS_ENABLED, "true"}, + {Options::CHANGELOG_PRODUCER, "input"}}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, schema, partition_keys, primary_keys, options)); + ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema)); } { std::map options = {{Options::BUCKET, "2"}, @@ -866,6 +877,41 @@ TEST(SchemaValidationTest, ValidateInvalidConfiguration) { "Can not set changelog-producer on table without primary keys, please " "define primary keys."); } + { + std::map options = { + {Options::CHANGELOG_PRODUCER, "input"}, + {Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE, "true"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, + /*primary_keys=*/{"f0"}, options)); + ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), + "'changelog-producer.row-deduplicate' is only valid for 'lookup' or " + "'full-compaction' changelog producer"); + } + { + std::map options = { + {Options::CHANGELOG_PRODUCER, "input"}, + {Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE_IGNORE_FIELDS, "f1"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, + /*primary_keys=*/{"f0"}, options)); + ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), + "'changelog-producer.row-deduplicate-ignore-fields' is only valid when " + "'changelog-producer.row-deduplicate' is true"); + } + { + std::map options = { + {Options::CHANGELOG_PRODUCER, "input"}, + {Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE, "true"}, + {Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE_IGNORE_FIELDS, "missing"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, + /*primary_keys=*/{"f0"}, options)); + ASSERT_NOK_WITH_MSG( + SchemaValidation::ValidateTableSchema(*table_schema), + "Fields [\"missing\"] configured in " + "'changelog-producer.row-deduplicate-ignore-fields' can not be found in table schema"); + } { auto invalid_field = arrow::field("_SEQUENCE_NUMBER", arrow::int64()); arrow::FieldVector invalid_fields = fields; @@ -897,15 +943,18 @@ TEST(SchemaValidationTest, ValidateInvalidConfiguration) { TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, /*primary_keys=*/{"f0"}, options)); ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), - "C++ Paimon does not support changelog-producer yet"); + "Only support 'none' and 'lookup' changelog-producer on FIRST_ROW " + "merge engine"); } { - std::map options = {{Options::CHANGELOG_PRODUCER, "lookup"}}; + std::map options = { + {Options::CHANGELOG_PRODUCER, "full-compaction"}}; ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, /*primary_keys=*/{"f0"}, options)); - ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), - "C++ Paimon does not support changelog-producer yet"); + ASSERT_NOK_WITH_MSG( + SchemaValidation::ValidateTableSchema(*table_schema), + "C++ Paimon only supports 'none', 'input' and 'lookup' changelog-producer now."); } // test for row tracking { @@ -1182,6 +1231,16 @@ TEST(SchemaValidationTest, TestMapSharedShreddingCompression) { "MAP shared-shredding only supports none/lz4/zstd compression, but " "file.compression.per.level.1 is snappy."); } + { + auto options = base_options; + options[Options::CHANGELOG_FILE_COMPRESSION] = "snappy"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options)); + ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), + "MAP shared-shredding only supports none/lz4/zstd compression, but " + "changelog-file.compression is snappy."); + } { auto options = base_options; options.erase("fields.f1.map.storage-layout"); @@ -1233,6 +1292,16 @@ TEST(SchemaValidationTest, TestMapSharedShreddingFileFormat) { "MAP shared-shredding only supports parquet/orc file formats, but " "file.format.per.level.1 is avro."); } + { + auto options = base_options; + options[Options::CHANGELOG_FILE_FORMAT] = "avro"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options)); + ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), + "MAP shared-shredding only supports parquet/orc file formats, but " + "changelog-file.format is avro."); + } } TEST(SchemaValidationTest, TestMapSharedShreddingRejectsPostponeBucketMode) { diff --git a/src/paimon/core/table/source/data_table_stream_scan.cpp b/src/paimon/core/table/source/data_table_stream_scan.cpp index 4c6d4df11..1657c9c7b 100644 --- a/src/paimon/core/table/source/data_table_stream_scan.cpp +++ b/src/paimon/core/table/source/data_table_stream_scan.cpp @@ -26,6 +26,7 @@ #include "paimon/core/options/changelog_producer.h" #include "paimon/core/table/bucket_mode.h" #include "paimon/core/table/source/plan_impl.h" +#include "paimon/core/table/source/snapshot/changelog_follow_up_scanner.h" #include "paimon/core/table/source/snapshot/delta_follow_up_scanner.h" #include "paimon/core/table/source/snapshot/follow_up_scanner.h" #include "paimon/core/table/source/snapshot/snapshot_reader.h" @@ -55,10 +56,15 @@ Result> DataTableStreamScan::CreatePlan() { Result> DataTableStreamScan::TryFirstPlan() { std::shared_ptr scan_result; - if (core_options_.GetChangelogProducer() == ChangelogProducer::LOOKUP) { - return Status::NotImplemented("do not support lookup changelog producer"); - } else if (core_options_.GetChangelogProducer() == ChangelogProducer::FULL_COMPACTION) { + if (core_options_.GetChangelogProducer() == ChangelogProducer::FULL_COMPACTION) { return Status::NotImplemented("do not support full compaction changelog producer"); + } else if (core_options_.GetChangelogProducer() == ChangelogProducer::LOOKUP) { + // Level-0 files will be compacted later to produce changelog records. Exclude them from + // the initial full scan so that the same changes are not emitted both in the full phase + // and again in the incremental changelog phase. + snapshot_reader_->WithLevelFilter([](int32_t level) -> bool { return level > 0; }); + PAIMON_ASSIGN_OR_RAISE(scan_result, starting_scanner_->Scan(snapshot_reader_)); + snapshot_reader_->WithLevelFilter([](int32_t) -> bool { return true; }); } else { PAIMON_ASSIGN_OR_RAISE(scan_result, starting_scanner_->Scan(snapshot_reader_)); } @@ -85,6 +91,9 @@ Result> DataTableStreamScan::NextPlan() { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, follow_up_scanner_->Scan(snapshot.value(), snapshot_reader_)); next_snapshot_id_.value()++; + if (plan->Splits().empty()) { + continue; + } return plan; } else { next_snapshot_id_.value()++; @@ -122,8 +131,19 @@ Result> DataTableStreamScan::GetNextSnapshot( Status DataTableStreamScan::InitScanner() { PAIMON_ASSIGN_OR_RAISE(starting_scanner_, CreateStartingScanner(/*is_streaming=*/true)); - follow_up_scanner_ = std::make_shared(); - return Status::OK(); + switch (core_options_.GetChangelogProducer()) { + case ChangelogProducer::NONE: + follow_up_scanner_ = std::make_shared(); + return Status::OK(); + case ChangelogProducer::INPUT: + case ChangelogProducer::LOOKUP: + follow_up_scanner_ = std::make_shared(); + return Status::OK(); + case ChangelogProducer::FULL_COMPACTION: + return Status::NotImplemented("do not support full compaction changelog producer"); + default: + return Status::NotImplemented("unknown changelog producer"); + } } } // namespace paimon diff --git a/src/paimon/core/table/source/scan_mode.h b/src/paimon/core/table/source/scan_mode.h index c236aad69..a01fde506 100644 --- a/src/paimon/core/table/source/scan_mode.h +++ b/src/paimon/core/table/source/scan_mode.h @@ -26,10 +26,10 @@ enum class ScanMode { ALL = 0, /// Only scan newly changed files of a snapshot. - DELTA = 1 + DELTA = 1, /// Only scan changelog files of a snapshot. - /* CHANGELOG = 2 */ + CHANGELOG = 2 }; } // namespace paimon diff --git a/src/paimon/core/table/source/snapshot/changelog_follow_up_scanner.h b/src/paimon/core/table/source/snapshot/changelog_follow_up_scanner.h new file mode 100644 index 000000000..ee2b73dfd --- /dev/null +++ b/src/paimon/core/table/source/snapshot/changelog_follow_up_scanner.h @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include "paimon/core/table/source/snapshot/follow_up_scanner.h" +#include "paimon/logging.h" + +namespace paimon { + +/// Follow-up scanner for snapshots containing changelog manifests. +class ChangelogFollowUpScanner : public FollowUpScanner { + public: + ChangelogFollowUpScanner() : logger_(Logger::GetLogger("ChangelogFollowUpScanner")) {} + + bool NeedScanSnapshot(const Snapshot& snapshot) const override { + if (snapshot.ChangelogManifestList()) { + return true; + } + PAIMON_LOG_DEBUG(logger_, "Snapshot #%ld has no changelog, check the next snapshot.", + snapshot.Id()); + return false; + } + + Result> Scan( + const Snapshot& snapshot, + const std::shared_ptr& snapshot_reader) const override { + return snapshot_reader->WithMode(ScanMode::CHANGELOG)->WithSnapshot(snapshot)->Read(); + } + + private: + std::unique_ptr logger_; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/source/snapshot/snapshot_reader_test.cpp b/src/paimon/core/table/source/snapshot/snapshot_reader_test.cpp index f3174fc6e..427d68fe2 100644 --- a/src/paimon/core/table/source/snapshot/snapshot_reader_test.cpp +++ b/src/paimon/core/table/source/snapshot/snapshot_reader_test.cpp @@ -30,6 +30,8 @@ #include "paimon/core/index/index_file_handler.h" #include "paimon/core/index/index_file_meta.h" #include "paimon/core/io/data_file_meta.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/source/snapshot/changelog_follow_up_scanner.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/index_file_path_factories.h" #include "paimon/fs/local/local_file_system.h" @@ -112,4 +114,23 @@ TEST_F(SnapshotReaderTest, GetDeletionFilesOverwritesDuplicateDataFileName) { EXPECT_EQ(deletion_files[0]->cardinality, std::optional(4)); } +TEST_F(SnapshotReaderTest, ChangelogFollowUpScannerSkipsSnapshotsWithoutChangelog) { + auto create_snapshot = [](const std::optional& changelog_manifest_list) { + return Snapshot( + /*id=*/1, /*schema_id=*/0, /*base_manifest_list=*/"", + /*base_manifest_list_size=*/std::nullopt, /*delta_manifest_list=*/"", + /*delta_manifest_list_size=*/std::nullopt, changelog_manifest_list, + /*changelog_manifest_list_size=*/std::nullopt, + /*index_manifest=*/std::nullopt, /*commit_user=*/"user", /*commit_identifier=*/1, + Snapshot::CommitKind::Append(), /*time_millis=*/0, /*total_record_count=*/0, + /*delta_record_count=*/0, /*changelog_record_count=*/std::nullopt, + /*watermark=*/std::nullopt, /*statistics=*/std::nullopt, + /*properties=*/std::nullopt, /*next_row_id=*/std::nullopt); + }; + + ChangelogFollowUpScanner scanner; + ASSERT_FALSE(scanner.NeedScanSnapshot(create_snapshot(std::nullopt))); + ASSERT_TRUE(scanner.NeedScanSnapshot(create_snapshot("changelog-manifest-list"))); +} + } // namespace paimon::test diff --git a/test/inte/scan_inte_test.cpp b/test/inte/scan_inte_test.cpp index b33fcf36b..f626dba6b 100644 --- a/test/inte/scan_inte_test.cpp +++ b/test/inte/scan_inte_test.cpp @@ -1273,9 +1273,9 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamWithAndPredicate) { .value()); std::vector>> expected_data_splits = { - {}, {expected_data_split1_2}, {expected_data_split2_1}, {}, {expected_data_split4_1}}; + {}, {expected_data_split1_2}, {expected_data_split2_1}, {expected_data_split4_1}}; - std::vector> expected_snapshot_ids = {std::nullopt, 1, 2, 3, 4}; + std::vector> expected_snapshot_ids = {std::nullopt, 1, 2, 4}; CheckStreamScanResult(table_scan.get(), expected_snapshot_ids, expected_data_splits); } diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index 397566d03..43ed0d1fb 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -90,6 +90,25 @@ class WriteAndReadInteTest return new_options; } + std::string LookupTempDirectory() const { + return PathUtil::JoinPath(test_dir_, "tmp"); + } + + Result> CreateLookupTestHelper( + const std::shared_ptr& schema, + const std::map& options) const { + return TestHelper::Create(test_dir_, schema, /*partition_keys=*/{}, + /*primary_keys=*/{"pk"}, options, + /*is_streaming_mode=*/true, /*ignore_if_exists=*/false, + LookupTempDirectory()); + } + + Result> CreateLookupTestHelper( + const std::string& table_path, const std::map& options) const { + return TestHelper::Create(table_path, options, /*is_streaming_mode=*/true, + LookupTempDirectory()); + } + Status WriteNextSchema(const std::vector& fields, int32_t highest_field_id, const std::map& options) const { return TestHelper::WriteNextSchema(dir_->GetFileSystem(), @@ -129,6 +148,33 @@ class WriteAndReadInteTest return file_system->AtomicStore(schema_path, std::string(buffer.GetString())); } + Status CompactAndCommit(const std::string& table_path, + const std::map& options, + int64_t commit_identifier) const { + WriteContextBuilder write_context_builder(table_path, "commit_user"); + write_context_builder.WithTempDirectory(LookupTempDirectory()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr write_context, + write_context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_write, + FileStoreWrite::Create(std::move(write_context))); + PAIMON_RETURN_NOT_OK(file_store_write->Compact(/*partition=*/{}, /*bucket=*/0, + /*full_compaction=*/true)); + PAIMON_ASSIGN_OR_RAISE( + std::vector> compact_messages, + file_store_write->PrepareCommit(/*wait_compaction=*/true, commit_identifier)); + PAIMON_RETURN_NOT_OK(file_store_write->Close()); + if (compact_messages.empty()) { + return Status::Invalid("expected compaction commit messages"); + } + CommitContextBuilder commit_context_builder(table_path, "commit_user"); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, + commit_context_builder.SetOptions(options).Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, + FileStoreCommit::Create(std::move(commit_context))); + return file_store_commit->Commit(compact_messages, commit_identifier); + } + Result ReadAndCheckProjectedResult(const std::map& options, const std::vector& read_fields, const std::shared_ptr& expected_type, @@ -630,6 +676,594 @@ TEST_P(WriteAndReadInteTest, TestPKSimple) { ASSERT_TRUE(success); } +TEST_P(WriteAndReadInteTest, TestInputChangelogStreamRead) { + arrow::FieldVector fields = { + arrow::field("pk", arrow::utf8()), + arrow::field("value", arrow::int32()), + }; + auto [file_format, file_system] = GetParam(); + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "input"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN( + auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), /*partition_keys=*/{}, + /*primary_keys=*/{"pk"}, options, /*is_streaming_mode=*/true)); + + ASSERT_OK_AND_ASSIGN(std::vector> initial_splits, + helper->NewScan(StartupMode::Latest(), /*snapshot_id=*/std::nullopt)); + ASSERT_TRUE(initial_splits.empty()); + + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch, + TestHelper::MakeRecordBatch( + arrow::struct_(fields), R"([["Alice", 10], ["Bob", 20], ["Alice", 11], ["Bob", 21]])", + /*partition_map=*/{}, /*bucket=*/0, + {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::UPDATE_BEFORE, + RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::DELETE})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(std::vector> changelog_splits, helper->Scan()); + ASSERT_TRUE(changelog_splits.empty()); + ASSERT_OK_AND_ASSIGN(changelog_splits, helper->Scan()); + ASSERT_FALSE(changelog_splits.empty()); + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + fields[0], + fields[1], + }); + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(expected_type, changelog_splits, + R"([[0, "Alice", 10], [2, "Alice", 11], + [1, "Bob", 20], [3, "Bob", 21]])")); + ASSERT_TRUE(success); +} + +TEST_P(WriteAndReadInteTest, TestLookupChangelogStreamRead) { + auto [file_format, file_system] = GetParam(); + arrow::FieldVector fields = { + arrow::field("pk", arrow::utf8()), + arrow::field("value", arrow::int32()), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"}, + }; + ASSERT_OK_AND_ASSIGN(auto helper, CreateLookupTestHelper(arrow::schema(fields), options)); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr initial_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 10]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(initial_batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + // Move the initial value to a high level so the next compaction must look it up. + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/1)); + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, CreateLookupTestHelper(table_path, options)); + + ASSERT_OK_AND_ASSIGN(std::vector> initial_splits, + helper->NewScan(StartupMode::Latest(), /*snapshot_id=*/std::nullopt)); + ASSERT_TRUE(initial_splits.empty()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr update_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 20]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(update_batch), /*commit_identifier=*/2, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/3)); + + ASSERT_OK_AND_ASSIGN(std::vector> changelog_splits, helper->Scan()); + ASSERT_FALSE(changelog_splits.empty()); + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + fields[0], + fields[1], + }); + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(expected_type, changelog_splits, + R"([[1, "Alice", 10], [2, "Alice", 20]])")); + ASSERT_TRUE(success); +} + +TEST_P(WriteAndReadInteTest, TestLookupChangelogInitialFullScanExcludesLevelZero) { + auto [file_format, file_system] = GetParam(); + arrow::FieldVector fields = { + arrow::field("pk", arrow::utf8()), + arrow::field("value", arrow::int32()), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"}, + }; + ASSERT_OK_AND_ASSIGN(auto helper, CreateLookupTestHelper(arrow::schema(fields), options)); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr initial_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 10]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(initial_batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/1)); + + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr update_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 20]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(update_batch), /*commit_identifier=*/2, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(std::vector> initial_full_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_FALSE(initial_full_splits.empty()); + for (const auto& split : initial_full_splits) { + auto data_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(data_split); + for (const auto& file : data_split->DataFiles()) { + ASSERT_GT(file->level, 0); + } + } + + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + fields[0], + fields[1], + }); + ASSERT_OK_AND_ASSIGN( + bool success, + helper->ReadAndCheckResult(expected_type, initial_full_splits, R"([[0, "Alice", 10]])")); + ASSERT_TRUE(success); + + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/3)); + + ASSERT_OK_AND_ASSIGN(std::vector> changelog_splits, helper->Scan()); + ASSERT_FALSE(changelog_splits.empty()); + ASSERT_OK_AND_ASSIGN(bool changelog_success, + helper->ReadAndCheckResult(expected_type, changelog_splits, + R"([[1, "Alice", 10], [2, "Alice", 20]])")); + ASSERT_TRUE(changelog_success); +} + +TEST_P(WriteAndReadInteTest, TestLookupChangelogInsertUpdateDelete) { + auto [file_format, file_system] = GetParam(); + arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()), + arrow::field("value", arrow::int32())}; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"}}; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, CreateLookupTestHelper(arrow::schema(fields), options)); + ASSERT_OK_AND_ASSIGN( + auto initial_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 10], ["Bob", 20]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(initial_batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/1)); + + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN(auto initial_splits, + helper->NewScan(StartupMode::Latest(), /*snapshot_id=*/std::nullopt)); + ASSERT_TRUE(initial_splits.empty()); + ASSERT_OK_AND_ASSIGN( + auto change_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([["Alice", 11], ["Bob", 0], ["Carol", 30], ["Dave", 0]])", + /*partition_map=*/{}, /*bucket=*/0, + {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::DELETE, + RecordBatch::RowKind::INSERT, RecordBatch::RowKind::DELETE})); + ASSERT_OK(helper->WriteAndCommit(std::move(change_batch), /*commit_identifier=*/2, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/3)); + + ASSERT_OK_AND_ASSIGN(auto changelog_splits, helper->Scan()); + ASSERT_FALSE(changelog_splits.empty()); + auto expected_type = + arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), fields[0], fields[1]}); + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(expected_type, changelog_splits, + R"([[1, "Alice", 10], [2, "Alice", 11], + [3, "Bob", 20], [0, "Carol", 30]])")); + ASSERT_TRUE(success); +} + +TEST_P(WriteAndReadInteTest, TestLookupChangelogWithFirstRow) { + auto [file_format, file_system] = GetParam(); + arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()), + arrow::field("value", arrow::int32())}; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"}, + {Options::MERGE_ENGINE, "first-row"}}; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, CreateLookupTestHelper(arrow::schema(fields), options)); + ASSERT_OK_AND_ASSIGN(auto initial_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 10]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(initial_batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/1)); + + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN(auto initial_splits, + helper->NewScan(StartupMode::Latest(), /*snapshot_id=*/std::nullopt)); + ASSERT_TRUE(initial_splits.empty()); + ASSERT_OK_AND_ASSIGN( + auto change_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 20], ["Bob", 30]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(change_batch), /*commit_identifier=*/2, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/3)); + + ASSERT_OK_AND_ASSIGN(auto changelog_splits, helper->Scan()); + ASSERT_FALSE(changelog_splits.empty()); + auto expected_type = + arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), fields[0], fields[1]}); + ASSERT_OK_AND_ASSIGN(bool success, helper->ReadAndCheckResult(expected_type, changelog_splits, + R"([[0, "Bob", 30]])")); + ASSERT_TRUE(success); +} + +TEST_P(WriteAndReadInteTest, TestLookupChangelogWithDeletionVector) { + auto [file_format, file_system] = GetParam(); + arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()), + arrow::field("value", arrow::int32())}; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"}, + {Options::DELETION_VECTORS_ENABLED, "true"}}; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, CreateLookupTestHelper(arrow::schema(fields), options)); + ASSERT_OK_AND_ASSIGN(auto initial_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 10]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(initial_batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/1)); + + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN(auto initial_splits, + helper->NewScan(StartupMode::Latest(), /*snapshot_id=*/std::nullopt)); + ASSERT_TRUE(initial_splits.empty()); + ASSERT_OK_AND_ASSIGN(auto update_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 20]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(update_batch), /*commit_identifier=*/2, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/3)); + + ASSERT_OK_AND_ASSIGN(auto changelog_splits, helper->Scan()); + ASSERT_FALSE(changelog_splits.empty()); + auto expected_type = + arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), fields[0], fields[1]}); + ASSERT_OK_AND_ASSIGN(bool changelog_success, + helper->ReadAndCheckResult(expected_type, changelog_splits, + R"([[1, "Alice", 10], [2, "Alice", 20]])")); + ASSERT_TRUE(changelog_success); + + ASSERT_OK_AND_ASSIGN(auto batch_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt, + /*is_streaming=*/false)); + ASSERT_OK_AND_ASSIGN(bool batch_success, helper->ReadAndCheckResult(expected_type, batch_splits, + R"([[0, "Alice", 20]])")); + ASSERT_TRUE(batch_success); +} + +TEST_P(WriteAndReadInteTest, TestLookupChangelogRowDeduplicate) { + auto [file_format, file_system] = GetParam(); + arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()), + arrow::field("value", arrow::int32())}; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, + {Options::CHANGELOG_PRODUCER, "lookup"}, + {Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE, "true"}}; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, CreateLookupTestHelper(arrow::schema(fields), options)); + ASSERT_OK_AND_ASSIGN(auto initial_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 10]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(initial_batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/1)); + + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN(auto initial_splits, + helper->NewScan(StartupMode::Latest(), /*snapshot_id=*/std::nullopt)); + ASSERT_TRUE(initial_splits.empty()); + ASSERT_OK_AND_ASSIGN(auto write_helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN(auto unchanged_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 10]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(write_helper->WriteAndCommit(std::move(unchanged_batch), /*commit_identifier=*/2, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/3)); + ASSERT_OK_AND_ASSIGN(auto empty_splits, helper->Scan()); + ASSERT_TRUE(empty_splits.empty()); + + write_helper.reset(); + ASSERT_OK_AND_ASSIGN(write_helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN(auto changed_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 20]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(write_helper->WriteAndCommit(std::move(changed_batch), /*commit_identifier=*/4, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/5)); + ASSERT_OK_AND_ASSIGN(auto changelog_splits, helper->Scan()); + ASSERT_FALSE(changelog_splits.empty()); + auto expected_type = + arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), fields[0], fields[1]}); + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(expected_type, changelog_splits, + R"([[1, "Alice", 10], [2, "Alice", 20]])")); + ASSERT_TRUE(success); +} + +TEST_P(WriteAndReadInteTest, TestLookupChangelogRowDeduplicateIgnoreFields) { + auto [file_format, file_system] = GetParam(); + arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()), + arrow::field("value", arrow::int32()), + arrow::field("ignored", arrow::int32())}; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, + {Options::CHANGELOG_PRODUCER, "lookup"}, + {Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE, "true"}, + {Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE_IGNORE_FIELDS, "ignored"}}; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, CreateLookupTestHelper(arrow::schema(fields), options)); + ASSERT_OK_AND_ASSIGN(auto initial_batch, TestHelper::MakeRecordBatch( + arrow::struct_(fields), R"([["Alice", 10, 100]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(initial_batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/1)); + + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN(auto initial_splits, + helper->NewScan(StartupMode::Latest(), /*snapshot_id=*/std::nullopt)); + ASSERT_TRUE(initial_splits.empty()); + ASSERT_OK_AND_ASSIGN(auto write_helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN( + auto ignored_change_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 10, 200]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(write_helper->WriteAndCommit(std::move(ignored_change_batch), + /*commit_identifier=*/2, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/3)); + ASSERT_OK_AND_ASSIGN(auto empty_splits, helper->Scan()); + ASSERT_TRUE(empty_splits.empty()); + + write_helper.reset(); + ASSERT_OK_AND_ASSIGN(write_helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN( + auto real_change_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 20, 300]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(write_helper->WriteAndCommit(std::move(real_change_batch), /*commit_identifier=*/4, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/5)); + ASSERT_OK_AND_ASSIGN(auto changelog_splits, helper->Scan()); + ASSERT_FALSE(changelog_splits.empty()); + auto expected_type = arrow::struct_( + {arrow::field("_VALUE_KIND", arrow::int8()), fields[0], fields[1], fields[2]}); + ASSERT_OK_AND_ASSIGN(bool success, helper->ReadAndCheckResult( + expected_type, changelog_splits, + R"([[1, "Alice", 10, 200], [2, "Alice", 20, 300]])")); + ASSERT_TRUE(success); +} + +TEST_P(WriteAndReadInteTest, TestChangelogWithSchemaEvolution) { + auto [file_format, file_system] = GetParam(); + arrow::FieldVector fields_v0 = {arrow::field("pk", arrow::utf8()), + arrow::field("value", arrow::int32())}; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"}}; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto write_helper, + CreateLookupTestHelper(arrow::schema(fields_v0), options)); + ASSERT_OK_AND_ASSIGN(auto initial_batch, TestHelper::MakeRecordBatch( + arrow::struct_(fields_v0), R"([["Alice", 10]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(write_helper->WriteAndCommit(std::move(initial_batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/1)); + + ASSERT_OK_AND_ASSIGN(auto scan_helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN(auto initial_splits, + scan_helper->NewScan(StartupMode::Latest(), /*snapshot_id=*/std::nullopt)); + ASSERT_TRUE(initial_splits.empty()); + + write_helper.reset(); + ASSERT_OK_AND_ASSIGN(write_helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN( + auto batch_v0, TestHelper::MakeRecordBatch(arrow::struct_(fields_v0), R"([["Alice", 11]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(write_helper->WriteAndCommit(std::move(batch_v0), /*commit_identifier=*/2, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/3)); + + arrow::FieldVector fields_v1 = {arrow::field("pk", arrow::utf8()), + arrow::field("value", arrow::int32()), + arrow::field("extra", arrow::utf8())}; + ASSERT_OK(WriteNextSchema( + {DataField(0, fields_v1[0]), DataField(1, fields_v1[1]), DataField(2, fields_v1[2])}, + /*highest_field_id=*/2, options)); + write_helper.reset(); + ASSERT_OK_AND_ASSIGN(write_helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN(auto batch_v1, TestHelper::MakeRecordBatch( + arrow::struct_(fields_v1), R"([["Alice", 12, "v1"]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(write_helper->WriteAndCommit(std::move(batch_v1), /*commit_identifier=*/4, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/5)); + + arrow::FieldVector fields_v2 = {arrow::field("pk", arrow::utf8()), + arrow::field("value", arrow::int32()), + arrow::field("renamed_extra", arrow::utf8())}; + ASSERT_OK(WriteNextSchema( + {DataField(0, fields_v2[0]), DataField(1, fields_v2[1]), DataField(2, fields_v2[2])}, + /*highest_field_id=*/2, options)); + write_helper.reset(); + ASSERT_OK_AND_ASSIGN(write_helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN(auto batch_v2, TestHelper::MakeRecordBatch( + arrow::struct_(fields_v2), R"([["Alice", 13, "v2"]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(write_helper->WriteAndCommit(std::move(batch_v2), /*commit_identifier=*/6, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/7)); + + arrow::FieldVector fields_v3 = {arrow::field("pk", arrow::utf8()), + arrow::field("value", arrow::int64()), + arrow::field("renamed_extra", arrow::utf8())}; + ASSERT_OK(WriteNextSchema( + {DataField(0, fields_v3[0]), DataField(1, fields_v3[1]), DataField(2, fields_v3[2])}, + /*highest_field_id=*/2, options)); + write_helper.reset(); + ASSERT_OK_AND_ASSIGN(write_helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN(auto batch_v3, TestHelper::MakeRecordBatch( + arrow::struct_(fields_v3), R"([["Alice", 14, "v3"]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(write_helper->WriteAndCommit(std::move(batch_v3), /*commit_identifier=*/8, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/9)); + + auto expected_type = arrow::struct_( + {arrow::field("_VALUE_KIND", arrow::int8()), fields_v3[0], fields_v3[1], fields_v3[2]}); + std::vector expected_data = { + R"([[1, "Alice", 10, null], [2, "Alice", 11, null]])", + R"([[1, "Alice", 11, null], [2, "Alice", 12, "v1"]])", + R"([[1, "Alice", 12, "v1"], [2, "Alice", 13, "v2"]])", + R"([[1, "Alice", 13, "v2"], [2, "Alice", 14, "v3"]])"}; + for (int64_t schema_id = 0; schema_id < static_cast(expected_data.size()); + schema_id++) { + ASSERT_OK_AND_ASSIGN(auto changelog_splits, scan_helper->Scan()); + ASSERT_FALSE(changelog_splits.empty()); + for (const auto& split : changelog_splits) { + auto data_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(data_split); + for (const auto& file : data_split->DataFiles()) { + ASSERT_EQ(file->schema_id, schema_id); + } + } + ASSERT_OK_AND_ASSIGN(bool success, scan_helper->ReadAndCheckResult( + expected_type, changelog_splits, + expected_data[static_cast(schema_id)])); + ASSERT_TRUE(success); + } +} + +TEST_P(WriteAndReadInteTest, TestLookupChangelogWithExternalPath) { + auto [file_format, file_system] = GetParam(); + if (file_system == "jindo") { + return; + } + std::unique_ptr external_dir = UniqueTestDirectory::Create(file_system); + ASSERT_TRUE(external_dir); + arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()), + arrow::field("value", arrow::int32())}; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, + {Options::CHANGELOG_PRODUCER, "lookup"}, + {Options::DATA_FILE_EXTERNAL_PATHS, "FILE://" + external_dir->Str()}, + {Options::DATA_FILE_EXTERNAL_PATHS_STRATEGY, "round-robin"}}; + ASSERT_OK_AND_ASSIGN(auto helper, CreateLookupTestHelper(arrow::schema(fields), options)); + ASSERT_OK_AND_ASSIGN(auto initial_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 10]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(initial_batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/1)); + + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN(auto initial_splits, + helper->NewScan(StartupMode::Latest(), /*snapshot_id=*/std::nullopt)); + ASSERT_TRUE(initial_splits.empty()); + ASSERT_OK_AND_ASSIGN(auto update_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 20]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(update_batch), /*commit_identifier=*/2, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/3)); + + ASSERT_OK_AND_ASSIGN(auto changelog_splits, helper->Scan()); + ASSERT_FALSE(changelog_splits.empty()); + bool found_external_changelog = false; + for (const auto& split : changelog_splits) { + auto data_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(data_split); + for (const auto& file : data_split->DataFiles()) { + ASSERT_TRUE(file->external_path.has_value()); + ASSERT_TRUE(StringUtils::StartsWith(PathUtil::GetName(file->file_name), "changelog-")); + ASSERT_OK_AND_ASSIGN( + bool exists, external_dir->GetFileSystem()->Exists(file->external_path.value())); + ASSERT_TRUE(exists); + found_external_changelog = true; + } + } + ASSERT_TRUE(found_external_changelog); + + auto expected_type = + arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), fields[0], fields[1]}); + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(expected_type, changelog_splits, + R"([[1, "Alice", 10], [2, "Alice", 20]])")); + ASSERT_TRUE(success); +} + TEST_P(WriteAndReadInteTest, TestNestedType) { arrow::FieldVector fields = { arrow::field("f1", arrow::map(arrow::int8(), arrow::int16())), @@ -2963,23 +3597,7 @@ TEST_P(WriteAndReadInteTest, TestAppendMapStorageLayoutSharedShreddingToDefaultC ASSERT_OK(helper->WriteAndCommit(std::move(batch_v1_file3), commit_identifier++, /*expected_commit_messages=*/std::nullopt)); - WriteContextBuilder write_context_builder(table_path, "commit_user"); - ASSERT_OK_AND_ASSIGN( - auto write_context, - write_context_builder.SetOptions(options_v1).WithStreamingMode(true).Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); - ASSERT_OK(file_store_write->Compact(/*partition=*/{}, /*bucket=*/0, - /*full_compaction=*/true)); - ASSERT_OK_AND_ASSIGN(auto compact_messages, file_store_write->PrepareCommit( - /*wait_compaction=*/true, commit_identifier)); - ASSERT_FALSE(compact_messages.empty()); - - CommitContextBuilder commit_context_builder(table_path, "commit_user"); - ASSERT_OK_AND_ASSIGN(auto commit_context, - commit_context_builder.SetOptions(options_v1).Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); - ASSERT_OK(file_store_commit->Commit(compact_messages, commit_identifier)); + ASSERT_OK(CompactAndCommit(table_path, options_v1, commit_identifier)); arrow::FieldVector expected_fields = fields; expected_fields.insert(expected_fields.begin(), arrow::field("_VALUE_KIND", arrow::int8()));