diff --git a/include/paimon/defs.h b/include/paimon/defs.h index e05faefd8..98dc8eb82 100644 --- a/include/paimon/defs.h +++ b/include/paimon/defs.h @@ -164,7 +164,8 @@ struct PAIMON_EXPORT Options { /// Default value is 8MB. static const char MANIFEST_TARGET_FILE_SIZE[]; - /// "manifest.format" - Specify the message format of manifest files. + /// "manifest.format" - Deprecated compatibility option for reading legacy manifests. + /// Avro is the only writable manifest format; non-Avro formats are read-only. /// Default value is avro. static const char MANIFEST_FORMAT[]; diff --git a/src/paimon/common/utils/string_utils.cpp b/src/paimon/common/utils/string_utils.cpp index 869e184d0..dd361f87c 100644 --- a/src/paimon/common/utils/string_utils.cpp +++ b/src/paimon/common/utils/string_utils.cpp @@ -37,6 +37,14 @@ bool IsTrimCharacter(unsigned char c) { return c <= 0x20; } +bool IsJavaWhitespace(uint32_t code_point) { + return (code_point >= 0x0009 && code_point <= 0x000d) || + (code_point >= 0x001c && code_point <= 0x0020) || code_point == 0x1680 || + (code_point >= 0x2000 && code_point <= 0x2006) || + (code_point >= 0x2008 && code_point <= 0x200a) || code_point == 0x2028 || + code_point == 0x2029 || code_point == 0x205f || code_point == 0x3000; +} + char ToAsciiLower(unsigned char c) { return c >= 'A' && c <= 'Z' ? static_cast(c + ('a' - 'A')) : static_cast(c); } @@ -85,18 +93,54 @@ bool StringUtils::EndsWith(const std::string& str, const std::string& suffix) { size_t s2 = suffix.size(); return (s1 >= s2) && (str.compare(s1 - s2, s2, suffix) == 0); } -bool StringUtils::IsNullOrWhitespaceOnly(const std::string& str) { - if (str.empty()) { - return true; - } - for (char c : str) { - if (!std::isspace(static_cast(c))) { + +bool StringUtils::IsBlank(std::string_view str) { + size_t offset = 0; + while (offset < str.size()) { + const auto first = static_cast(str[offset]); + uint32_t code_point = 0; + size_t length = 0; + if (first <= 0x7f) { + code_point = first; + length = 1; + } else if (first >= 0xc2 && first <= 0xdf) { + code_point = first & 0x1f; + length = 2; + } else if (first >= 0xe0 && first <= 0xef) { + code_point = first & 0x0f; + length = 3; + } else if (first >= 0xf0 && first <= 0xf4) { + code_point = first & 0x07; + length = 4; + } else { + return false; + } + if (offset + length > str.size()) { + return false; + } + for (size_t i = 1; i < length; ++i) { + const auto continuation = static_cast(str[offset + i]); + if ((continuation & 0xc0) != 0x80) { + return false; + } + code_point = (code_point << 6) | (continuation & 0x3f); + } + if ((length == 3 && code_point < 0x800) || (length == 4 && code_point < 0x10000) || + (code_point >= 0xd800 && code_point <= 0xdfff) || code_point > 0x10ffff) { + return false; + } + if (!IsJavaWhitespace(code_point)) { return false; } + offset += length; } return true; } +bool StringUtils::IsNullOrWhitespaceOnly(const std::string& str) { + return IsBlank(str); +} + void StringUtils::Trim(std::string* str) { auto first = std::find_if_not(str->begin(), str->end(), [](unsigned char c) { return IsTrimCharacter(c); }); diff --git a/src/paimon/common/utils/string_utils.h b/src/paimon/common/utils/string_utils.h index 7681a8d9e..32025de0c 100644 --- a/src/paimon/common/utils/string_utils.h +++ b/src/paimon/common/utils/string_utils.h @@ -31,6 +31,7 @@ #include #include #include +#include #include #include @@ -96,6 +97,10 @@ class PAIMON_EXPORT StringUtils { static bool EndsWith(const std::string& str, const std::string& suffix); + /// Returns true if the string is empty or contains only characters recognized by Java + /// Character.isWhitespace. + static bool IsBlank(std::string_view str); + static bool IsNullOrWhitespaceOnly(const std::string& str); static void Trim(std::string* str); diff --git a/src/paimon/common/utils/string_utils_test.cpp b/src/paimon/common/utils/string_utils_test.cpp index a4f230781..874eb7a09 100644 --- a/src/paimon/common/utils/string_utils_test.cpp +++ b/src/paimon/common/utils/string_utils_test.cpp @@ -21,6 +21,7 @@ #include #include +#include #include "gtest/gtest.h" #include "paimon/status.h" @@ -209,6 +210,21 @@ TEST_F(StringUtilsTest, TestIsNullOrWhitespaceOnly) { auto ret = StringUtils::IsNullOrWhitespaceOnly(str); ASSERT_TRUE(ret); } + ASSERT_TRUE(StringUtils::IsNullOrWhitespaceOnly(u8"\u3000\u2000")); +} + +TEST_F(StringUtilsTest, TestIsBlank) { + const std::vector blank_strings = { + "", " ", " ", "\t", "\n", "\r", + "\r\n", " \t\n\r ", u8"\u1680", u8"\u2000", u8"\u3000", u8" \t\u3000\u2000\n"}; + for (const std::string& blank : blank_strings) { + ASSERT_TRUE(StringUtils::IsBlank(blank)) << blank; + } + + ASSERT_FALSE(StringUtils::IsBlank("user1")); + ASSERT_FALSE(StringUtils::IsBlank(" user1 ")); + ASSERT_FALSE(StringUtils::IsBlank(u8"\u00a0")); + ASSERT_FALSE(StringUtils::IsBlank(std::string("\xc0\x80", 2))); } TEST_F(StringUtilsTest, TestToLowerCase) { diff --git a/src/paimon/core/append/append_compact_coordinator.cpp b/src/paimon/core/append/append_compact_coordinator.cpp index b6afc15c4..b6dd4a0d6 100644 --- a/src/paimon/core/append/append_compact_coordinator.cpp +++ b/src/paimon/core/append/append_compact_coordinator.cpp @@ -133,17 +133,17 @@ Result> CreateFileStoreScan( const std::shared_ptr& path_factory, const std::shared_ptr& scan_filter, const std::shared_ptr& executor, const std::shared_ptr& pool) { - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr manifest_list, - ManifestList::Create(core_options.GetFileSystem(), core_options.GetManifestFormat(), - core_options.GetManifestCompression(), path_factory, - core_options.GetCache(), pool)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr manifest_file, - ManifestFile::Create(core_options.GetFileSystem(), core_options.GetManifestFormat(), - core_options.GetManifestCompression(), path_factory, - core_options.GetManifestTargetFileSize(), pool, core_options, - partition_schema)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_format, + core_options.GetManifestFormat(/*write=*/false)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_list, + ManifestList::Create(core_options.GetFileSystem(), manifest_format, + core_options.GetManifestCompression(), path_factory, + core_options.GetCache(), pool)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_file, + ManifestFile::Create(core_options.GetFileSystem(), manifest_format, + core_options.GetManifestCompression(), path_factory, + core_options.GetManifestTargetFileSize(), pool, + core_options, partition_schema)); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr scan, AppendOnlyFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp index 71ba73deb..be86d200f 100644 --- a/src/paimon/core/core_options.cpp +++ b/src/paimon/core/core_options.cpp @@ -610,7 +610,7 @@ struct CoreOptions::Impl { // Parse manifest file configurations: format, compression, merge, and compaction thresholds. Status ParseManifestOptions(const ConfigParser& parser) { - // Parse manifest.format - manifest file format, default "avro" + // Parse manifest.format for reading legacy manifests; only avro is writable. PAIMON_RETURN_NOT_OK(parser.ParseObject( Options::MANIFEST_FORMAT, /*default_identifier=*/"avro", &manifest_file_format)); // Parse manifest.compression - manifest file compression, default "zstd" @@ -1143,7 +1143,13 @@ std::string CoreOptions::GetPartitionDefaultName() const { return impl_->partition_default_name; } -std::shared_ptr CoreOptions::GetManifestFormat() const { +Result> CoreOptions::GetManifestFormat(bool write) const { + const std::string& identifier = impl_->manifest_file_format->Identifier(); + if (write && identifier != "avro") { + return Status::Invalid(fmt::format( + "manifest.format '{}' is read-only; only 'avro' can be used for writing manifests", + identifier)); + } return impl_->manifest_file_format; } diff --git a/src/paimon/core/core_options.h b/src/paimon/core/core_options.h index 345958e1a..29d6097f5 100644 --- a/src/paimon/core/core_options.h +++ b/src/paimon/core/core_options.h @@ -93,7 +93,9 @@ class PAIMON_EXPORT CoreOptions { int64_t GetCompactionFileSize(bool has_primary_key) const; std::string GetPartitionDefaultName() const; - std::shared_ptr GetManifestFormat() const; + /// Return the configured manifest format for the requested access mode. + /// Non-Avro formats are supported only for reading legacy manifests. + Result> GetManifestFormat(bool write) const; const std::string& GetManifestCompression() const; int32_t GetManifestMergeMinCount() const; int64_t GetManifestFullCompactionThresholdSize() const; diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp index c424b9cfe..150f4a1d2 100644 --- a/src/paimon/core/core_options_test.cpp +++ b/src/paimon/core/core_options_test.cpp @@ -37,7 +37,9 @@ namespace paimon::test { TEST(CoreOptionsTest, TestDefaultValue) { ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({})); - ASSERT_EQ(core_options.GetManifestFormat()->Identifier(), "avro"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr manifest_format, + core_options.GetManifestFormat(/*write=*/false)); + ASSERT_EQ(manifest_format->Identifier(), "avro"); ASSERT_EQ(core_options.GetFileFormat()->Identifier(), "parquet"); ASSERT_EQ(nullptr, core_options.GetChangelogFileFormat()); ASSERT_EQ(core_options.GetWriteFileFormat(0)->Identifier(), "parquet"); @@ -193,6 +195,23 @@ TEST(CoreOptionsTest, TestDefaultValue) { ASSERT_EQ(BucketFunctionType::DEFAULT, core_options.GetBucketFunctionType()); } +TEST(CoreOptionsTest, GetManifestFormatForReadAndWrite) { + ASSERT_OK_AND_ASSIGN(CoreOptions default_options, CoreOptions::FromMap({})); + ASSERT_OK(default_options.GetManifestFormat(/*write=*/true)); + + ASSERT_OK_AND_ASSIGN(CoreOptions avro_options, + CoreOptions::FromMap({{Options::MANIFEST_FORMAT, "AvRo"}})); + ASSERT_OK(avro_options.GetManifestFormat(/*write=*/true)); + + ASSERT_OK_AND_ASSIGN(CoreOptions legacy_options, + CoreOptions::FromMap({{Options::MANIFEST_FORMAT, "orc"}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr legacy_manifest_format, + legacy_options.GetManifestFormat(/*write=*/false)); + ASSERT_EQ(legacy_manifest_format->Identifier(), "orc"); + ASSERT_NOK_WITH_MSG(legacy_options.GetManifestFormat(/*write=*/true), + "manifest.format 'orc' is read-only"); +} + TEST(CoreOptionsTest, TestFromMap) { std::map options = { {Options::FILE_SYSTEM, "Local"}, @@ -339,7 +358,8 @@ TEST(CoreOptionsTest, TestFromMap) { ASSERT_EQ(core_options.GetWriteFileFormat(1)->Identifier(), "orc"); ASSERT_EQ(core_options.GetWriteFileFormat(3)->Identifier(), "parquet"); - auto manifest_format = core_options.GetManifestFormat(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr manifest_format, + core_options.GetManifestFormat(/*write=*/false)); ASSERT_EQ(manifest_format->Identifier(), "avro"); ASSERT_EQ(3, core_options.GetBucket()); diff --git a/src/paimon/core/global_index/global_index_scan_impl.cpp b/src/paimon/core/global_index/global_index_scan_impl.cpp index e4b3e7b75..10c7079b0 100644 --- a/src/paimon/core/global_index/global_index_scan_impl.cpp +++ b/src/paimon/core/global_index/global_index_scan_impl.cpp @@ -82,9 +82,11 @@ Result> GlobalIndexScanImpl::Create( std::shared_ptr path_factory = file_store_path_factory->CreateGlobalIndexFileFactory(); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_format, + options.GetManifestFormat(/*write=*/false)); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr index_manifest_file, - IndexManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), + IndexManifestFile::Create(options.GetFileSystem(), manifest_format, options.GetManifestCompression(), file_store_path_factory, options.GetBucket(), pool, options)); auto index_file_handler = std::make_unique( diff --git a/src/paimon/core/index/index_file_handler_test.cpp b/src/paimon/core/index/index_file_handler_test.cpp index a23591f94..305579e27 100644 --- a/src/paimon/core/index/index_file_handler_test.cpp +++ b/src/paimon/core/index/index_file_handler_test.cpp @@ -70,11 +70,13 @@ class IndexFileHandlerTest : public testing::Test { global_index_external_path, /*index_file_in_data_file_dir=*/core_options.IndexFileInDataFileDir(), memory_pool_)); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr index_manifest_file, - IndexManifestFile::Create( - core_options.GetFileSystem(), core_options.GetManifestFormat(), - core_options.GetManifestCompression(), path_factory, - core_options.GetBucket(), memory_pool_, core_options)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_format, + core_options.GetManifestFormat(/*write=*/false)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr index_manifest_file, + IndexManifestFile::Create(core_options.GetFileSystem(), manifest_format, + core_options.GetManifestCompression(), path_factory, + core_options.GetBucket(), memory_pool_, core_options)); auto path_factories = std::make_shared(path_factory); return std::make_unique( core_options.GetFileSystem(), std::move(index_manifest_file), path_factories, diff --git a/src/paimon/core/mergetree/compact/aggregate/field_listagg_agg.h b/src/paimon/core/mergetree/compact/aggregate/field_listagg_agg.h index a6d7ed5c4..a6dc04b7c 100644 --- a/src/paimon/core/mergetree/compact/aggregate/field_listagg_agg.h +++ b/src/paimon/core/mergetree/compact/aggregate/field_listagg_agg.h @@ -20,10 +20,12 @@ #include #include +#include #include #include #include "paimon/common/data/data_define.h" +#include "paimon/common/utils/string_utils.h" #include "paimon/core/core_options.h" #include "paimon/core/mergetree/compact/aggregate/field_aggregator.h" @@ -60,15 +62,18 @@ class FieldListaggAgg : public FieldAggregator { const VariantType& input_field) override { bool accumulator_null = DataDefine::IsVariantNull(accumulator); bool input_null = DataDefine::IsVariantNull(input_field); - if (accumulator_null || input_null) { - return accumulator_null ? input_field : accumulator; + if (input_null) { + return accumulator; } - std::string_view acc_str = DataDefine::GetStringView(accumulator); std::string_view in_str = DataDefine::GetStringView(input_field); - if (in_str.empty()) { + if (StringUtils::IsBlank(in_str)) { return accumulator; } - if (acc_str.empty()) { + if (accumulator_null) { + return input_field; + } + std::string_view acc_str = DataDefine::GetStringView(accumulator); + if (StringUtils::IsBlank(acc_str)) { return input_field; } @@ -95,9 +100,7 @@ class FieldListaggAgg : public FieldAggregator { size_t pos = remaining.find(delimiter_); std::string_view token = (pos == std::string_view::npos) ? remaining : remaining.substr(0, pos); - if (!token.empty()) { - seen.insert(token); - } + seen.insert(token); if (pos == std::string_view::npos) { break; } @@ -113,7 +116,7 @@ class FieldListaggAgg : public FieldAggregator { size_t pos = remaining.find(delimiter_); std::string_view token = (pos == std::string_view::npos) ? remaining : remaining.substr(0, pos); - if (!token.empty() && seen.insert(token).second) { + if (!StringUtils::IsBlank(token) && seen.insert(token).second) { result.append(delimiter_); result.append(token); } diff --git a/src/paimon/core/mergetree/compact/aggregate/field_listagg_agg_test.cpp b/src/paimon/core/mergetree/compact/aggregate/field_listagg_agg_test.cpp index 3578cdaac..1efac34ed 100644 --- a/src/paimon/core/mergetree/compact/aggregate/field_listagg_agg_test.cpp +++ b/src/paimon/core/mergetree/compact/aggregate/field_listagg_agg_test.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include "arrow/type_fwd.h" #include "gtest/gtest.h" @@ -88,13 +89,41 @@ TEST_F(FieldListaggAggTest, TestEmptyString) { auto ret = agg->Agg(std::string_view(""), std::string_view("world")).value(); ASSERT_EQ(DataDefine::GetVariantValue(ret), "world"); } - // both empty -> return input (which is empty) + // blank input -> return accumulator (which is empty) { auto ret = agg->Agg(std::string_view(""), std::string_view("")).value(); ASSERT_EQ(DataDefine::GetVariantValue(ret), ""); } } +TEST_F(FieldListaggAggTest, TestBlankStrings) { + ASSERT_OK_AND_ASSIGN(auto agg, MakeAgg()); + + const std::vector blank_strings = {"", + " ", + " ", + "\t", + "\n", + "\r", + "\r\n", + " \t\n\r ", + u8"\u3000", + u8"\u2000", + u8" \t\u3000\u2000\n"}; + for (const std::string& blank : blank_strings) { + auto ret = agg->Agg(std::string_view("user1"), std::string_view(blank)).value(); + ASSERT_EQ(DataDefine::GetVariantValue(ret), "user1"); + } + + // A blank accumulator must not add a leading delimiter. + auto ret = agg->Agg(std::string_view(u8"\u3000\t"), std::string_view("user1")).value(); + ASSERT_EQ(DataDefine::GetVariantValue(ret), "user1"); + + // A blank input must not turn a null accumulator into a non-null value. + ret = agg->Agg(NullType(), std::string_view(u8" \t\u3000")).value(); + ASSERT_TRUE(DataDefine::IsVariantNull(ret)); +} + TEST_F(FieldListaggAggTest, TestMultipleAccumulation) { ASSERT_OK_AND_ASSIGN(auto agg, MakeAgg()); @@ -113,6 +142,15 @@ TEST_F(FieldListaggAggTest, TestDistinct) { ASSERT_EQ(DataDefine::GetVariantValue(ret), "a;b;c"); } +TEST_F(FieldListaggAggTest, TestDistinctIgnoresBlankTokens) { + ASSERT_OK_AND_ASSIGN(auto agg, MakeAgg(",", true)); + + auto ret = + agg->Agg(std::string_view("user1"), std::string_view(u8" ,user2,\t,\u3000,user1,\u2000")) + .value(); + ASSERT_EQ(DataDefine::GetVariantValue(ret), "user1,user2"); +} + TEST_F(FieldListaggAggTest, TestDistinctNoDuplicates) { ASSERT_OK_AND_ASSIGN(auto agg, MakeAgg(" ", true)); diff --git a/src/paimon/core/operation/append_only_file_store_write.cpp b/src/paimon/core/operation/append_only_file_store_write.cpp index f660093a2..2a5affa16 100644 --- a/src/paimon/core/operation/append_only_file_store_write.cpp +++ b/src/paimon/core/operation/append_only_file_store_write.cpp @@ -120,14 +120,16 @@ Status AppendOnlyFileStoreWrite::RefreshCommittedSnapshot(int64_t snapshot_id) { Result> AppendOnlyFileStoreWrite::CreateFileStoreScan( const std::shared_ptr& scan_filter) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_format, + options_.GetManifestFormat(/*write=*/false)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr manifest_list, - ManifestList::Create(options_.GetFileSystem(), options_.GetManifestFormat(), + ManifestList::Create(options_.GetFileSystem(), manifest_format, options_.GetManifestCompression(), file_store_path_factory_, options_.GetCache(), pool_)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr manifest_file, - ManifestFile::Create(options_.GetFileSystem(), options_.GetManifestFormat(), + ManifestFile::Create(options_.GetFileSystem(), manifest_format, options_.GetManifestCompression(), file_store_path_factory_, options_.GetManifestTargetFileSize(), pool_, options_, partition_schema_)); diff --git a/src/paimon/core/operation/expire_snapshots_test.cpp b/src/paimon/core/operation/expire_snapshots_test.cpp index ffdd30feb..364d2b502 100644 --- a/src/paimon/core/operation/expire_snapshots_test.cpp +++ b/src/paimon/core/operation/expire_snapshots_test.cpp @@ -75,14 +75,16 @@ class ExpireSnapshotsTest : public testing::Test { test_data_path_ = "tmp"; path_factory_ = CreateFactory(test_data_path_); + ASSERT_OK_AND_ASSIGN(std::shared_ptr manifest_format, + options.GetManifestFormat(/*write=*/false)); ASSERT_OK_AND_ASSIGN( manifest_list_, - ManifestList::Create(fs_, options.GetManifestFormat(), options.GetManifestCompression(), + ManifestList::Create(fs_, manifest_format, options.GetManifestCompression(), path_factory_, options.GetCache(), mem_pool_)); ASSERT_OK_AND_ASSIGN( manifest_file_, - ManifestFile::Create(fs_, options.GetManifestFormat(), options.GetManifestCompression(), + ManifestFile::Create(fs_, manifest_format, options.GetManifestCompression(), path_factory_, options.GetManifestTargetFileSize(), mem_pool_, options, partition_schema_)); } diff --git a/src/paimon/core/operation/file_store_commit.cpp b/src/paimon/core/operation/file_store_commit.cpp index ad0942ade..dbe4d2c22 100644 --- a/src/paimon/core/operation/file_store_commit.cpp +++ b/src/paimon/core/operation/file_store_commit.cpp @@ -127,6 +127,8 @@ Result> FileStoreCommit::Create( assert(options.GetFileSystem()); assert(options.GetFileFormat()); PAIMON_RETURN_NOT_OK(FileStoreCommitImpl::ValidateCommitOptions(options)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_format, + options.GetManifestFormat(/*write=*/true)); PAIMON_ASSIGN_OR_RAISE(bool is_object_store, FileSystem::IsObjectStore(root_path)); if (is_object_store && !ctx->UseRESTCatalogCommit() && @@ -152,24 +154,22 @@ Result> FileStoreCommit::Create( global_index_external_path, options.IndexFileInDataFileDir(), ctx->GetMemoryPool())); auto snapshot_manager = std::make_shared(options.GetFileSystem(), root_path); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr manifest_list, - ManifestList::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), path_factory, options.GetCache(), - ctx->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_list, + ManifestList::Create(options.GetFileSystem(), manifest_format, + options.GetManifestCompression(), path_factory, + options.GetCache(), ctx->GetMemoryPool())); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr partition_schema, FieldMapping::GetPartitionSchema(arrow_schema, table_schema.value()->PartitionKeys())); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr manifest_file, - ManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), path_factory, - options.GetManifestTargetFileSize(), ctx->GetMemoryPool(), options, - partition_schema)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_file, + ManifestFile::Create(options.GetFileSystem(), manifest_format, + options.GetManifestCompression(), path_factory, + options.GetManifestTargetFileSize(), + ctx->GetMemoryPool(), options, partition_schema)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr index_manifest_file, - IndexManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), + IndexManifestFile::Create(options.GetFileSystem(), manifest_format, options.GetManifestCompression(), path_factory, options.GetBucket(), ctx->GetMemoryPool(), options)); diff --git a/src/paimon/core/operation/file_store_commit_impl_test.cpp b/src/paimon/core/operation/file_store_commit_impl_test.cpp index b1b8677f8..e4422ffbc 100644 --- a/src/paimon/core/operation/file_store_commit_impl_test.cpp +++ b/src/paimon/core/operation/file_store_commit_impl_test.cpp @@ -379,7 +379,7 @@ class FileStoreCommitImplTest : public testing::Test { TEST_F(FileStoreCommitImplTest, TestCommit) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -408,7 +408,7 @@ TEST_F(FileStoreCommitImplTest, TestRESTCatalogCommit) { TimezoneGuard guard("Asia/Shanghai"); CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .UseRESTCatalogCommit(true) @@ -466,7 +466,7 @@ TEST_F(FileStoreCommitImplTest, TestRESTCatalogCommit) { TEST_F(FileStoreCommitImplTest, TestSnapshotSequenceMaxPropertyMergedOnCommit) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .AddOption(Options::WRITE_SEQUENCE_NUMBER_INIT_MODE, "snapshot") @@ -514,7 +514,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryTenTimes) FileSystemFactory::Get("gmock_fs", table_path, {})); CommitContextBuilder context_builder(table_path, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::COMMIT_MAX_RETRIES, "10") .AddOption(Options::COMMIT_MIN_RETRY_WAIT, "1ms") @@ -548,7 +548,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryTenTimes) } TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryOnce) { - std::string test_data_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/"; + std::string test_data_path = paimon::test::GetDataDir() + "/parquet/append_09.db/append_09/"; auto dir = UniqueTestDirectory::Create(); std::string table_path = dir->Str(); ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); @@ -556,7 +556,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryOnce) { FileSystemFactory::Get("gmock_fs", table_path, {})); CommitContextBuilder context_builder(table_path, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::COMMIT_MIN_RETRY_WAIT, "1ms") .AddOption(Options::COMMIT_MAX_RETRY_WAIT, "1ms") @@ -591,7 +591,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryOnce) { std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + - "/orc/append_09.db/append_09/commit_messages/commit_messages-01", + "/parquet/append_09.db/append_09/commit_messages/commit_messages-01", /*version=*/3); ASSERT_GT(msgs.size(), 0); ASSERT_OK(commit->Commit(msgs)); @@ -606,7 +606,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryOnce) { } TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActuallySucceed) { - std::string test_data_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/"; + std::string test_data_path = paimon::test::GetDataDir() + "/parquet/append_09.db/append_09/"; auto dir = UniqueTestDirectory::Create(); std::string table_path = dir->Str(); ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); @@ -614,7 +614,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua FileSystemFactory::Get("gmock_fs", table_path, {})); CommitContextBuilder context_builder(table_path, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .WithFileSystem(fs) .Finish()); @@ -632,7 +632,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + - "/orc/append_09.db/append_09/commit_messages/commit_messages-01", + "/parquet/append_09.db/append_09/commit_messages/commit_messages-01", /*version=*/3); ASSERT_GT(msgs.size(), 0); ASSERT_NOK(commit->Commit(msgs, /*commit_identifier=*/1)); @@ -642,7 +642,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua CommitContextBuilder context_builder_2(table_path, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context_2, - context_builder_2.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder_2.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .WithFileSystem(fs) .Finish()); @@ -657,7 +657,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua })); std::vector> msgs_2 = GetCommitMessages(paimon::test::GetDataDir() + - "/orc/append_09.db/append_09/commit_messages/commit_messages-02", + "/parquet/append_09.db/append_09/commit_messages/commit_messages-02", /*version=*/3); ASSERT_OK(commit_2->Commit(msgs_2, /*commit_identifier=*/2)); ASSERT_OK_AND_ASSIGN(exist, file_system_->Exists(new_snapshot_7)); @@ -667,7 +667,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua TEST_F(FileStoreCommitImplTest, TestCommitWithSameMsgs) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "5kb") .AddOption(Options::MANIFEST_MERGE_MIN_COUNT, "2") .AddOption(Options::FILE_SYSTEM, "local") @@ -731,7 +731,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithSameMsgs) { TEST_F(FileStoreCommitImplTest, TestCommitMultipleTimes) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -803,7 +803,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitMultipleTimes) { TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatest) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -861,7 +861,7 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatest) { TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestNoLatestSnapshotReturnsError) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -873,7 +873,7 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestNoLatestSnapshotReturnsErr TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestTargetNotExistReturnsError) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -908,7 +908,7 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestDeletionVectorOnlyChange) CommitContextBuilder context_builder(dv_table_path, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -978,7 +978,7 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestConcurrentConflictReturnsF FileSystemFactory::Get("gmock_fs", table_path_, {})); CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .WithFileSystem(fs) .Finish()); @@ -1021,7 +1021,7 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestConcurrentConflictReturnsF TEST_F(FileStoreCommitImplTest, TestCommitAndOverwriteWithNoPartitionKey) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -1067,7 +1067,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitAndOverwriteWithNoPartitionKey) { TEST_F(FileStoreCommitImplTest, TestCommitSuccessAfterIOException) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -1125,7 +1125,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitSuccessAfterIOException) { TEST_F(FileStoreCommitImplTest, TestCleanUpTmpManifests) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -1243,7 +1243,7 @@ TEST_F(FileStoreCommitImplTest, TestCleanUpTmpManifests) { TEST_F(FileStoreCommitImplTest, TestCommitWithIgnoreEmptyCommit) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .IgnoreEmptyCommit(true) @@ -1262,7 +1262,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIgnoreEmptyCommit) { TEST_F(FileStoreCommitImplTest, TestTryOverwrite) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .IgnoreEmptyCommit(true) @@ -1293,7 +1293,7 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwrite) { TEST_F(FileStoreCommitImplTest, TestTryOverwriteFromNothing) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .IgnoreEmptyCommit(true) @@ -1328,7 +1328,7 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteFromNothing) { TEST_F(FileStoreCommitImplTest, TestTryOverwriteWithProperties) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .IgnoreEmptyCommit(true) @@ -1356,7 +1356,7 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteWithProperties) { TEST_F(FileStoreCommitImplTest, TestTryOverwriteThenCommit) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .IgnoreEmptyCommit(true) @@ -1408,7 +1408,7 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteThenCommit) { TEST_F(FileStoreCommitImplTest, TestDropPartitionAndExpireSnapshot) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .AddOption(Options::SNAPSHOT_NUM_RETAINED_MIN, "1") @@ -1456,7 +1456,7 @@ TEST_F(FileStoreCommitImplTest, TestDropPartitionAndExpireSnapshot) { TEST_F(FileStoreCommitImplTest, TestDropMultiPartitionAndExpireSnapshot) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .AddOption(Options::SNAPSHOT_NUM_RETAINED_MIN, "1") @@ -1504,7 +1504,7 @@ TEST_F(FileStoreCommitImplTest, TestDropMultiPartitionAndExpireSnapshot) { TEST_F(FileStoreCommitImplTest, TestTruncateTable) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .IgnoreEmptyCommit(true) @@ -1538,7 +1538,7 @@ TEST_F(FileStoreCommitImplTest, TestTruncateTable) { TEST_F(FileStoreCommitImplTest, TestAbortDeletesDataAndIndexFiles) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); @@ -1587,7 +1587,7 @@ TEST_F(FileStoreCommitImplTest, TestAbortDeletesDataAndIndexFiles) { TEST_F(FileStoreCommitImplTest, AbortIgnoresMissingFilesAndFailsForNonImplMessage) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); @@ -1613,7 +1613,7 @@ TEST_F(FileStoreCommitImplTest, AbortIgnoresDeleteFailures) { // propagate the failure. CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); @@ -1654,7 +1654,7 @@ TEST_F(FileStoreCommitImplTest, TestTruncateEmptyTable) { // materialized. CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .IgnoreEmptyCommit(true) @@ -1680,7 +1680,7 @@ TEST_F(FileStoreCommitImplTest, TestTruncateEmptyTable) { TEST_F(FileStoreCommitImplTest, TestCreateManifestCommittable) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .IgnoreEmptyCommit(true) @@ -1701,7 +1701,7 @@ TEST_F(FileStoreCommitImplTest, TestCreateManifestCommittable) { TEST_F(FileStoreCommitImplTest, TestCollectChanges) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .AddOption(Options::BUCKET, "10") @@ -1745,7 +1745,7 @@ TEST_F(FileStoreCommitImplTest, TestCollectChanges) { TEST_F(FileStoreCommitImplTest, TestFilterCommitted) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -1777,7 +1777,7 @@ TEST_F(FileStoreCommitImplTest, TestFilterCommitted) { TEST_F(FileStoreCommitImplTest, TestFilterCommittedWithMultipleCommittables) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -1818,7 +1818,7 @@ TEST_F(FileStoreCommitImplTest, TestFilterCommittedWithMultipleCommittables) { TEST_F(FileStoreCommitImplTest, TestFilterCommittedRejectsDuplicateIdentifiers) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -1850,7 +1850,7 @@ TEST_F(FileStoreCommitImplTest, TestFilterCommittedRejectsDuplicateIdentifiers) TEST_F(FileStoreCommitImplTest, FilterAndCommit) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -1898,7 +1898,7 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommit) { TEST_F(FileStoreCommitImplTest, FilterAndCommitWithNotExistFile) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -1921,7 +1921,7 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitWithNotExistFile) { TEST_F(FileStoreCommitImplTest, FilterAndCommitWithCompactedChangelogFakePath) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -1956,7 +1956,7 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitWithCompactedChangelogFakePath) { TEST_F(FileStoreCommitImplTest, FilterAndCommitSkipCompactBeforeFileCheck) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -1990,7 +1990,7 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitSkipCompactBeforeFileCheck) { TEST_F(FileStoreCommitImplTest, TestOverwriteNonSpecifyPartition) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -2030,7 +2030,7 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteNonSpecifyPartition) { TEST_F(FileStoreCommitImplTest, TestCommitWithIndexFiles) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -2061,7 +2061,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIndexFiles) { TEST_F(FileStoreCommitImplTest, TestCommitWithGlobalIndexFilesChecksConflicts) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .AddOption(Options::ROW_TRACKING_ENABLED, "true") @@ -2094,7 +2094,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithGlobalIndexFilesChecksConflicts) { TEST_F(FileStoreCommitImplTest, TestCommitWithCompactIndexFiles) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .IgnoreEmptyCommit(true) @@ -2127,7 +2127,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithCompactIndexFiles) { TEST_F(FileStoreCommitImplTest, TestCommitWithDeletedIndexFiles) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -2165,7 +2165,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithDeletedIndexFiles) { TEST_F(FileStoreCommitImplTest, TestCommitWithCompactDeletedIndexFiles) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .IgnoreEmptyCommit(true) @@ -2205,7 +2205,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithCompactDeletedIndexFiles) { TEST_F(FileStoreCommitImplTest, TestOverwriteWithCompactIndexFiles) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -2239,7 +2239,7 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithCompactIndexFiles) { TEST_F(FileStoreCommitImplTest, TestFilterAndOverwrite) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -2295,7 +2295,7 @@ TEST_F(FileStoreCommitImplTest, TestFilterAndOverwrite) { TEST_F(FileStoreCommitImplTest, TestFilterAndOverwriteWithCompactIndexFiles) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -2334,7 +2334,7 @@ TEST_F(FileStoreCommitImplTest, TestFilterAndOverwriteWithCompactIndexFiles) { TEST_F(FileStoreCommitImplTest, TestOverwriteWithSpecifyPartition) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -2369,7 +2369,7 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithSpecifyPartition) { TEST_F(FileStoreCommitImplTest, TestOverwriteWithSameFile) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -2400,7 +2400,7 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithSameFile) { TEST_F(FileStoreCommitImplTest, TestAppendDiscardDuplicateFiles) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .AddOption(Options::COMMIT_DISCARD_DUPLICATE_FILES, "true") @@ -2433,7 +2433,7 @@ TEST_F(FileStoreCommitImplTest, TestAppendDiscardDuplicateFiles) { TEST_F(FileStoreCommitImplTest, TestCommitWithIOException) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -2466,7 +2466,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIOException) { io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); CommitContextBuilder context_builder2(tmp_table_path, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context2, - context_builder2.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder2.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -2497,7 +2497,7 @@ TEST_F(FileStoreCommitImplTest, TestObjectStoreAllowedWithRESTCatalogCommit) { CommitContextBuilder builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN( auto ctx, - builder.AddOption(Options::MANIFEST_FORMAT, "orc").UseRESTCatalogCommit(true).Finish()); + builder.AddOption(Options::MANIFEST_FORMAT, "avro").UseRESTCatalogCommit(true).Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(ctx))); auto msgs = @@ -2560,10 +2560,18 @@ TEST_F(FileStoreCommitImplTest, ValidateCommitOptionsAllowsManifestDeleteFileDro } } +TEST_F(FileStoreCommitImplTest, CreateRejectsReadOnlyManifestFormat) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc").Finish()); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), + "manifest.format 'orc' is read-only"); +} + TEST_F(FileStoreCommitImplTest, TestGetAllFilesKeepsValueStats) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::MANIFEST_DELETE_FILE_DROP_STATS, "true") .AddOption(Options::FILE_SYSTEM, "local") @@ -2593,7 +2601,7 @@ TEST_F(FileStoreCommitImplTest, TestGetAllFilesKeepsValueStats) { TEST_F(FileStoreCommitImplTest, TestOverwriteDropsDeleteFileStats) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::MANIFEST_DELETE_FILE_DROP_STATS, "true") .AddOption(Options::FILE_SYSTEM, "local") @@ -2669,7 +2677,7 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteDropsDeleteFileStats) { TEST_F(FileStoreCommitImplTest, DropPartitionWithEmptyPartitionsFails) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); @@ -2680,7 +2688,7 @@ TEST_F(FileStoreCommitImplTest, DropPartitionWithEmptyPartitionsFails) { TEST_F(FileStoreCommitImplTest, FilterAndCommitMultipleIdentifiersAndEmptyInput) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -2720,7 +2728,7 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitMultipleIdentifiersAndEmptyInput) TEST_F(FileStoreCommitImplTest, CheckFilesExistenceFailsForNonImplCommitMessage) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); @@ -2736,7 +2744,7 @@ TEST_F(FileStoreCommitImplTest, CheckFilesExistenceFailsForNonImplCommitMessage) TEST_F(FileStoreCommitImplTest, CheckFilesExistenceCollectsIndexFilePaths) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); @@ -2765,7 +2773,7 @@ TEST_F(FileStoreCommitImplTest, CheckFilesExistenceCollectsIndexFilePaths) { TEST_F(FileStoreCommitImplTest, OverwriteStaticPartitionValidatesFileOwnership) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .AddOption(Options::DYNAMIC_PARTITION_OVERWRITE, "false") @@ -2794,7 +2802,7 @@ TEST_F(FileStoreCommitImplTest, OverwriteStaticPartitionValidatesFileOwnership) TEST_F(FileStoreCommitImplTest, OverwriteWithChangelogFilesLogsWarning) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -2840,7 +2848,7 @@ TEST_F(FileStoreCommitImplTest, OverwriteUpgradesNonOverlappingPrimaryKeyFiles) CommitContextBuilder builder(pk_table_path, "test_user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - builder.AddOption(Options::MANIFEST_FORMAT, "orc") + builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::FILE_SYSTEM, "local") .AddOption(Options::OVERWRITE_UPGRADE, "true") .Finish()); @@ -2891,7 +2899,7 @@ TEST_F(FileStoreCommitImplTest, OverwriteUpgradesNonOverlappingPrimaryKeyFiles) TEST_F(FileStoreCommitImplTest, CommitWithAppendCommitCheckConflict) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .AppendCommitCheckConflict(true) @@ -2915,7 +2923,7 @@ TEST_F(FileStoreCommitImplTest, SnapshotSequenceMaxFallsBackToManifestScan) { { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -2931,7 +2939,7 @@ TEST_F(FileStoreCommitImplTest, SnapshotSequenceMaxFallsBackToManifestScan) { // the max sequence number is recomputed by scanning the base manifests. CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .AddOption(Options::WRITE_SEQUENCE_NUMBER_INIT_MODE, "snapshot") @@ -2961,7 +2969,7 @@ TEST_F(FileStoreCommitImplTest, SnapshotSequenceMaxFallsBackToManifestScan) { TEST_F(FileStoreCommitImplTest, FilterAndOverwriteWithSpecifiedPartition) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -2988,7 +2996,7 @@ TEST_F(FileStoreCommitImplTest, FilterAndOverwriteWithSpecifiedPartition) { TEST_F(FileStoreCommitImplTest, TryUpgradeReturnsInputWhenOverwriteUpgradeDisabled) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::FILE_SYSTEM, "local") .AddOption(Options::OVERWRITE_UPGRADE, "false") .Finish()); @@ -3022,7 +3030,7 @@ TEST_F(FileStoreCommitImplTest, TryUpgradeReturnsInputWhenEntryLevelAboveZero) { CommitContextBuilder builder(pk_table_path, "test_user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - builder.AddOption(Options::MANIFEST_FORMAT, "orc") + builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::FILE_SYSTEM, "local") .AddOption(Options::OVERWRITE_UPGRADE, "true") .Finish()); @@ -3043,7 +3051,7 @@ TEST_F(FileStoreCommitImplTest, TryUpgradeReturnsInputWhenEntryLevelAboveZero) { TEST_F(FileStoreCommitImplTest, CheckSameBucketFromSnapshotReturnsOkForEmptyDelta) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -3067,7 +3075,7 @@ TEST_F(FileStoreCommitImplTest, CheckSameBucketFromSnapshotReturnsOkForEmptyDelt TEST_F(FileStoreCommitImplTest, MaxSequenceNumberReturnsNulloptForEmptyManifests) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); @@ -3082,7 +3090,7 @@ TEST_F(FileStoreCommitImplTest, MaxSequenceNumberReturnsNulloptForEmptyManifests TEST_F(FileStoreCommitImplTest, RowIdCheckConflictSetsCheckSnapshotAndReturnsSelf) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); diff --git a/src/paimon/core/operation/file_store_commit_test.cpp b/src/paimon/core/operation/file_store_commit_test.cpp index b43e32158..270407714 100644 --- a/src/paimon/core/operation/file_store_commit_test.cpp +++ b/src/paimon/core/operation/file_store_commit_test.cpp @@ -77,7 +77,7 @@ TEST(FileStoreCommitTest, TestCreate) { CommitContextBuilder context_builder(table_path, "commit_user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -115,7 +115,7 @@ TEST(FileStoreCommitTest, TestAppendDvIndexShouldUseOverwriteCommitKind) { CommitContextBuilder context_builder(table_path, "commit_user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + context_builder.AddOption(Options::MANIFEST_FORMAT, "avro") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp index 6807ae35e..bf64a5dad 100644 --- a/src/paimon/core/operation/file_store_write.cpp +++ b/src/paimon/core/operation/file_store_write.cpp @@ -174,9 +174,11 @@ Result> FileStoreWrite::Create(std::unique_ptr dv_maintainer_factory; if (need_dv_maintainer_factory) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_format, + options.GetManifestFormat(/*write=*/true)); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr index_manifest_file, - IndexManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), + IndexManifestFile::Create(options.GetFileSystem(), manifest_format, options.GetManifestCompression(), file_store_path_factory, options.GetBucket(), ctx->GetMemoryPool(), options)); auto index_file_handler = std::make_shared( @@ -235,9 +237,11 @@ Result> FileStoreWrite::Create(std::unique_ptr dv_maintainer_factory; if (options.DeletionVectorsEnabled()) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_format, + options.GetManifestFormat(/*write=*/true)); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr index_manifest_file, - IndexManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), + IndexManifestFile::Create(options.GetFileSystem(), manifest_format, options.GetManifestCompression(), file_store_path_factory, options.GetBucket(), ctx->GetMemoryPool(), options)); auto index_file_handler = std::make_shared( diff --git a/src/paimon/core/operation/key_value_file_store_scan_test.cpp b/src/paimon/core/operation/key_value_file_store_scan_test.cpp index f9fd2ea2f..d08d1d494 100644 --- a/src/paimon/core/operation/key_value_file_store_scan_test.cpp +++ b/src/paimon/core/operation/key_value_file_store_scan_test.cpp @@ -70,12 +70,13 @@ class KeyValueFileStoreScanTest : public testing::Test { Result> CreateFileStoreScan( const std::string& table_path, const std::shared_ptr& scan_filter, int32_t table_schema_id, int32_t snapshot_id) const { - std::map options_map = {{Options::MANIFEST_FORMAT, "orc"}}; - PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options_map)); - auto fs = core_options.GetFileSystem(); + PAIMON_ASSIGN_OR_RAISE(CoreOptions bootstrap_options, CoreOptions::FromMap({})); + auto fs = bootstrap_options.GetFileSystem(); auto schema_manager = std::make_shared(fs, table_path); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_schema, schema_manager->ReadSchema(table_schema_id)); + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, + CoreOptions::FromMap(table_schema->Options())); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, @@ -91,7 +92,8 @@ class KeyValueFileStoreScanTest : public testing::Test { core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), pool_)); - auto manifest_file_format = core_options.GetManifestFormat(); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_file_format, + core_options.GetManifestFormat(/*write=*/false)); auto snapshot_manager = std::make_shared(fs, table_path); PAIMON_ASSIGN_OR_RAISE( diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp index 09755984d..c555b29c3 100644 --- a/src/paimon/core/operation/key_value_file_store_write.cpp +++ b/src/paimon/core/operation/key_value_file_store_write.cpp @@ -78,14 +78,16 @@ KeyValueFileStoreWrite::KeyValueFileStoreWrite( Result> KeyValueFileStoreWrite::CreateFileStoreScan( const std::shared_ptr& scan_filter) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_format, + options_.GetManifestFormat(/*write=*/false)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr manifest_list, - ManifestList::Create(options_.GetFileSystem(), options_.GetManifestFormat(), + ManifestList::Create(options_.GetFileSystem(), manifest_format, options_.GetManifestCompression(), file_store_path_factory_, options_.GetCache(), pool_)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr manifest_file, - ManifestFile::Create(options_.GetFileSystem(), options_.GetManifestFormat(), + ManifestFile::Create(options_.GetFileSystem(), manifest_format, options_.GetManifestCompression(), file_store_path_factory_, options_.GetManifestTargetFileSize(), pool_, options_, partition_schema_)); diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp index 35d938af7..074c9433f 100644 --- a/src/paimon/core/operation/key_value_file_store_write_test.cpp +++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp @@ -437,7 +437,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriterRestoreKeepsValueStats) { std::map options = { {Options::BUCKET, "1"}, {Options::FILE_FORMAT, "orc"}, - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::MANIFEST_DELETE_FILE_DROP_STATS, "true"}}; ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(dir->Str(), options)); ASSERT_OK(catalog->CreateDatabase("foo", {}, /*ignore_if_exists=*/false)); diff --git a/src/paimon/core/operation/orphan_files_cleaner.cpp b/src/paimon/core/operation/orphan_files_cleaner.cpp index ac69d5a37..2c4fab118 100644 --- a/src/paimon/core/operation/orphan_files_cleaner.cpp +++ b/src/paimon/core/operation/orphan_files_cleaner.cpp @@ -194,20 +194,20 @@ Result> OrphanFilesCleaner::Create( global_index_external_path, options.IndexFileInDataFileDir(), ctx->GetMemoryPool())); auto snapshot_manager = std::make_shared(options.GetFileSystem(), ctx->GetRootPath()); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr manifest_list, - ManifestList::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), path_factory, options.GetCache(), - ctx->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_format, + options.GetManifestFormat(/*write=*/false)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_list, + ManifestList::Create(options.GetFileSystem(), manifest_format, + options.GetManifestCompression(), path_factory, + options.GetCache(), ctx->GetMemoryPool())); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr partition_schema, FieldMapping::GetPartitionSchema(arrow_schema, table_schema.value()->PartitionKeys())); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr manifest_file, - ManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), path_factory, - options.GetManifestTargetFileSize(), ctx->GetMemoryPool(), options, - partition_schema)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_file, + ManifestFile::Create(options.GetFileSystem(), manifest_format, + options.GetManifestCompression(), path_factory, + options.GetManifestTargetFileSize(), + ctx->GetMemoryPool(), options, partition_schema)); return std::make_unique( ctx->GetMemoryPool(), ctx->GetExecutor(), arrow_schema, ctx->GetRootPath(), options, snapshot_manager, schema->PartitionKeys(), manifest_file, manifest_list, diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp index 8111a94e3..06209f7e1 100644 --- a/src/paimon/core/table/source/table_scan.cpp +++ b/src/paimon/core/table/source/table_scan.cpp @@ -90,7 +90,8 @@ class TableScanImpl { const std::shared_ptr& executor, const std::shared_ptr& memory_pool, const ScanContext* context) { auto fs = core_options.GetFileSystem(); - auto manifest_file_format = core_options.GetManifestFormat(); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_file_format, + core_options.GetManifestFormat(/*write=*/false)); std::string branch = BranchManager::NormalizeBranch(core_options.GetBranch()); auto snapshot_manager = std::make_shared(fs, context->GetPath(), branch); // TODO(liancheng.lsz): support fallback branch in scan @@ -172,11 +173,13 @@ class TableScanImpl { static Result> CreateIndexFileHandler( const CoreOptions& core_options, const std::shared_ptr& path_factory, const std::shared_ptr& memory_pool) { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr index_manifest_file, - IndexManifestFile::Create( - core_options.GetFileSystem(), core_options.GetManifestFormat(), - core_options.GetManifestCompression(), path_factory, - core_options.GetBucket(), memory_pool, core_options)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_format, + core_options.GetManifestFormat(/*write=*/false)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr index_manifest_file, + IndexManifestFile::Create(core_options.GetFileSystem(), manifest_format, + core_options.GetManifestCompression(), path_factory, + core_options.GetBucket(), memory_pool, core_options)); return std::make_unique( core_options.GetFileSystem(), std::move(index_manifest_file), std::make_shared(path_factory), diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp index 6523588e8..16c25ed02 100644 --- a/src/paimon/core/table/system/global_system_tables.cpp +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -211,11 +211,13 @@ Result AggregateFileStats(const std::shared_ptr core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), pool)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_format, + core_options.GetManifestFormat(/*write=*/false)); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr manifest_list, - ManifestList::Create(fs, core_options.GetManifestFormat(), - core_options.GetManifestCompression(), path_factory, - core_options.GetCache(), pool)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr manifest_list, + ManifestList::Create(fs, manifest_format, core_options.GetManifestCompression(), + path_factory, core_options.GetCache(), pool)); std::vector manifests; PAIMON_RETURN_NOT_OK(manifest_list->ReadDataManifests(*snapshot, &manifests)); @@ -223,11 +225,11 @@ Result AggregateFileStats(const std::shared_ptr PAIMON_ASSIGN_OR_RAISE( std::shared_ptr partition_schema, FieldMapping::GetPartitionSchema(arrow_schema, table_schema.PartitionKeys())); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr manifest_file, - ManifestFile::Create(fs, core_options.GetManifestFormat(), - core_options.GetManifestCompression(), path_factory, - core_options.GetManifestTargetFileSize(), pool, - core_options, partition_schema)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr manifest_file, + ManifestFile::Create(fs, manifest_format, core_options.GetManifestCompression(), + path_factory, core_options.GetManifestTargetFileSize(), pool, + core_options, partition_schema)); std::vector entries; for (const auto& manifest : manifests) { diff --git a/src/paimon/core/table/system/metadata_system_tables.cpp b/src/paimon/core/table/system/metadata_system_tables.cpp index 093994317..7096a65e9 100644 --- a/src/paimon/core/table/system/metadata_system_tables.cpp +++ b/src/paimon/core/table/system/metadata_system_tables.cpp @@ -228,10 +228,12 @@ Result> ReadDataManifests( const MetadataSystemTableContext& context, const Snapshot& snapshot, const std::shared_ptr& path_factory, const CoreOptions& core_options, const std::shared_ptr& pool) { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr manifest_list, - ManifestList::Create(context.fs, core_options.GetManifestFormat(), - core_options.GetManifestCompression(), path_factory, - core_options.GetCache(), pool)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_format, + core_options.GetManifestFormat(/*write=*/false)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr manifest_list, + ManifestList::Create(context.fs, manifest_format, core_options.GetManifestCompression(), + path_factory, core_options.GetCache(), pool)); std::vector manifests; // TODO(suxiaogang223): Align Java ReadAllManifests semantics by including changelog // manifests. ReadAllManifests currently delegates to ReadChangelogManifests, which returns @@ -249,10 +251,11 @@ Result> CreateManifestFile( PAIMON_ASSIGN_OR_RAISE( std::shared_ptr partition_schema, FieldMapping::GetPartitionSchema(arrow_schema, context.table_schema->PartitionKeys())); - return ManifestFile::Create(context.fs, core_options.GetManifestFormat(), - core_options.GetManifestCompression(), path_factory, - core_options.GetManifestTargetFileSize(), pool, core_options, - partition_schema); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_format, + core_options.GetManifestFormat(/*write=*/false)); + return ManifestFile::Create(context.fs, manifest_format, core_options.GetManifestCompression(), + path_factory, core_options.GetManifestTargetFileSize(), pool, + core_options, partition_schema); } Result> ReadLatestManifestEntries( diff --git a/src/paimon/format/parquet/parquet_format_defs.h b/src/paimon/format/parquet/parquet_format_defs.h index 8b205a09a..9271c7479 100644 --- a/src/paimon/format/parquet/parquet_format_defs.h +++ b/src/paimon/format/parquet/parquet_format_defs.h @@ -41,7 +41,7 @@ namespace paimon::parquet { static inline const char PARQUET_BLOCK_SIZE[] = "parquet.block.size"; static inline const char PARQUET_PAGE_SIZE[] = "parquet.page.size"; static inline const char PARQUET_DICTIONARY_PAGE_SIZE[] = "parquet.dictionary.page.size"; -static inline const char PARQUET_ENABLE_DICTIONARY[] = "parquet.enable-dictionary"; +static inline const char PARQUET_ENABLE_DICTIONARY[] = "parquet.enable.dictionary"; static inline const char PARQUET_WRITER_VERSION[] = "parquet.writer.version"; static inline const char PARQUET_WRITE_MAX_ROW_GROUP_LENGTH[] = "parquet.write.max-row-group-length"; diff --git a/src/paimon/format/parquet/parquet_writer_builder_test.cpp b/src/paimon/format/parquet/parquet_writer_builder_test.cpp index 3a95a8de3..2bbd3df21 100644 --- a/src/paimon/format/parquet/parquet_writer_builder_test.cpp +++ b/src/paimon/format/parquet/parquet_writer_builder_test.cpp @@ -77,6 +77,20 @@ TEST(ParquetWriterBuilderTest, PrepareWriterProperties) { ASSERT_EQ(3, properties->default_column_properties().compression_level()); } +TEST(ParquetWriterBuilderTest, PrepareWriterPropertiesWithDictionaryDisabled) { + arrow::FieldVector fields; + std::shared_ptr schema = arrow::schema(fields); + std::map options = { + {Options::FILE_FORMAT, "parquet"}, + {Options::MANIFEST_FORMAT, "parquet"}, + {"parquet.enable.dictionary", "false"}, + }; + ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options); + ASSERT_OK_AND_ASSIGN(std::shared_ptr<::parquet::WriterProperties> properties, + builder.PrepareWriterProperties("zstd")); + ASSERT_FALSE(properties->default_column_properties().dictionary_enabled()); +} + TEST(ParquetWriterBuilderTest, PrepareWriterPropertiesWithFileBlockSize) { arrow::FieldVector fields; std::shared_ptr schema = arrow::schema(fields); diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp index f5d5960b3..ac1489c8e 100644 --- a/test/inte/blob_table_inte_test.cpp +++ b/test/inte/blob_table_inte_test.cpp @@ -129,7 +129,7 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter } void CreateTable(const std::vector& partition_keys) const { - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -212,7 +212,7 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("view", true)}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -611,7 +611,7 @@ TEST_P(BlobTableInteTest, TestAppendTableWriteWithBlobAsDescriptorTrue) { BlobUtils::ToArrowField("blob", true)}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, {Options::BLOB_AS_DESCRIPTOR, "true"}, {Options::FILE_SYSTEM, "local"}}; @@ -652,7 +652,7 @@ TEST_P(BlobTableInteTest, TestAppendTableWriteWithBlobAsDescriptorFalse) { BlobUtils::ToArrowField("blob", true)}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, {Options::BLOB_AS_DESCRIPTOR, "false"}, {Options::FILE_SYSTEM, "local"}}; @@ -686,7 +686,7 @@ TEST_P(BlobTableInteTest, TestWriteNullOnMissingFile) { BlobUtils::ToArrowField("blob", true)}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, {Options::BLOB_AS_DESCRIPTOR, "true"}, {Options::BLOB_WRITE_NULL_ON_MISSING_FILE, "true"}, @@ -747,7 +747,7 @@ TEST_P(BlobTableInteTest, TestMissingFileFailsWriteWhenWriteNullDisabled) { BlobUtils::ToArrowField("blob", true)}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, {Options::BLOB_AS_DESCRIPTOR, "true"}, {Options::FILE_SYSTEM, "local"}}; @@ -778,7 +778,7 @@ TEST_P(BlobTableInteTest, TestWriteNullOnFetchFailure) { BlobUtils::ToArrowField("blob", true)}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, @@ -845,7 +845,7 @@ TEST_P(BlobTableInteTest, TestWriteNullOnFetchFailureCoversMissingFile) { BlobUtils::ToArrowField("blob", true)}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, @@ -947,7 +947,7 @@ TEST_P(BlobTableInteTest, TestBasic) { } TEST_P(BlobTableInteTest, TestBlobFilesAcrossSchemaIds) { - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -1123,7 +1123,7 @@ TEST_P(BlobTableInteTest, TestMultipleAppends) { TEST_P(BlobTableInteTest, TestDataEvolutionBlobOnlyWriteWithFirstRowId) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")}; - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -1205,7 +1205,7 @@ TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateFallback) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0", /*nullable=*/true)}; - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -1289,7 +1289,7 @@ TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateFallback) { TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateMultipleLayers) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")}; - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -1361,7 +1361,7 @@ TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateWithDeletionVectors) arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, {Options::DELETION_VECTORS_ENABLED, "true"}}; CreateTable(fields, /*partition_keys=*/{}, options); @@ -1444,7 +1444,7 @@ TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateWithDeletionVectors) TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateCompactedLayers) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")}; - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -1511,7 +1511,7 @@ TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateCompactedLayers) { TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateWithRowRanges) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")}; - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -1571,7 +1571,7 @@ TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateWithRowRanges) { TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateRowTrackingWithSubrangeLayer) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("b0", /*nullable=*/true)}; - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -1657,7 +1657,7 @@ TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateRowTrackingWithSubra TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateAllPlaceholderRowTracking) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("b0", /*nullable=*/true)}; - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -1713,7 +1713,7 @@ TEST_P(BlobTableInteTest, TestBlobValueEqualToPlaceholderSentinelBytes) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0", /*nullable=*/true)}; - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -1780,7 +1780,7 @@ TEST_P(BlobTableInteTest, TestBlobSentinelValueInBaseLayerDegradesToNull) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0", /*nullable=*/true)}; - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -1824,7 +1824,7 @@ TEST_P(BlobTableInteTest, TestUserSuppliedInternalPlaceholderOptionsIgnored) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0", /*nullable=*/true)}; - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -1851,7 +1851,7 @@ TEST_P(BlobTableInteTest, TestUserSuppliedInternalPlaceholderOptionsIgnored) { TEST_P(BlobTableInteTest, TestDataEvolutionBlobOnlyFirstCommitFailsWithoutFirstRowId) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")}; - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -2009,7 +2009,7 @@ TEST_P(BlobTableInteTest, TestMoreDataWithDataEvolution) { TEST_P(BlobTableInteTest, TestBlobWriteMultiRound) { std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::BLOB_TARGET_FILE_SIZE, "1000"}, {Options::TARGET_FILE_SIZE, "100"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; @@ -2052,7 +2052,7 @@ TEST_P(BlobTableInteTest, TestExternalPath) { std::string external_test_dir = external_dir->Str(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -2114,7 +2114,7 @@ TEST_P(BlobTableInteTest, TestPartitionWithPredicate) { auto file_format = GetParam(); std::vector partition_keys = {"f0"}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, {"parquet.write.max-row-group-length", "1"}}; @@ -2234,7 +2234,7 @@ TEST_P(BlobTableInteTest, TestPredicate) { return; } if (GetParam() == "mosaic") { - CreateTable(/*partition_keys=*/{}, {{Options::MANIFEST_FORMAT, "orc"}, + CreateTable(/*partition_keys=*/{}, {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -2574,7 +2574,7 @@ TEST_P(BlobTableInteTest, TestWithRowIdsSimple) { TEST_P(BlobTableInteTest, TestWithRowIdsForMultipleBlobFiles) { auto file_format = GetParam(); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1000"}, {Options::BLOB_TARGET_FILE_SIZE, "80"}, @@ -2685,7 +2685,7 @@ TEST_P(BlobTableInteTest, TestAppendTableWriteWithMultipleBlobFields) { BlobUtils::ToArrowField("blob1", true), BlobUtils::ToArrowField("blob2", true)}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, {Options::FILE_SYSTEM, "local"}}; @@ -2711,7 +2711,7 @@ TEST_P(BlobTableInteTest, TestAppendWriteWithNullBlob) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("blob", true)}; - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::BUCKET, "-1"}, {Options::FILE_SYSTEM, "local"}, @@ -2819,7 +2819,7 @@ TEST_P(BlobTableInteTest, TestBlobDescriptorField) { BlobUtils::ToArrowField("b1", true)}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, {Options::FILE_SYSTEM, "local"}}; @@ -2877,7 +2877,7 @@ TEST_P(BlobTableInteTest, TestBlobDescriptorFieldPartialInline) { BlobUtils::ToArrowField("b3", true)}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, {Options::FILE_SYSTEM, "local"}}; @@ -2940,7 +2940,7 @@ TEST_P(BlobTableInteTest, TestBlobDescriptorMultiCommitAndShuffledReadSchema) { BlobUtils::ToArrowField("b3", true)}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, {Options::FILE_SYSTEM, "local"}}; @@ -3068,7 +3068,7 @@ TEST_P(BlobTableInteTest, TestSharedShreddingWithBlobDataEvolution) { BlobUtils::ToArrowField("payload"), }; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -3128,7 +3128,7 @@ TEST_P(BlobTableInteTest, TestMultipleSharedShreddingMapsWithBlobDataEvolution) BlobUtils::ToArrowField("payload"), }; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -3189,7 +3189,7 @@ TEST_P(BlobTableInteTest, TestSharedShreddingMapOverrideWithBlobDataEvolution) { BlobUtils::ToArrowField("payload"), }; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -3248,7 +3248,7 @@ TEST_P(BlobTableInteTest, TestOrcMapStorageLayoutEvolutionWithBlobDataEvolution) BlobUtils::ToArrowField("payload"), }; std::map options_v0 = { - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, "orc"}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -3320,7 +3320,7 @@ TEST_P(BlobTableInteTest, TestDataEvolutionWithBlobDescriptorField) { BlobUtils::ToArrowField("b3", true)}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, {Options::FILE_SYSTEM, "local"}}; @@ -3437,7 +3437,7 @@ TEST_P(BlobTableInteTest, TestBlobDescriptorFieldWriteRawBytesDirectly) { BlobUtils::ToArrowField("b1", true)}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, GetParam()}, {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, {Options::FILE_SYSTEM, "local"}}; @@ -3475,7 +3475,7 @@ TEST_P(BlobTableInteTest, TestBlobViewFieldWithUpstreamTable) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("view", true)}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -3644,7 +3644,7 @@ TEST_P(BlobTableInteTest, TestForwardBlobViewReference) { // dynamically disabled can succeed on it. arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("view", true)}; - std::map source_options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map source_options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -3796,7 +3796,7 @@ TEST_P(BlobTableInteTest, TestBlobViewFieldWithUpstreamDescriptorBlob) { BlobUtils::ToArrowField("b1", true)}; auto upstream_schema = arrow::schema(upstream_fields); std::map upstream_options = { - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -3838,7 +3838,7 @@ TEST_P(BlobTableInteTest, TestBlobViewFieldWithUpstreamDescriptorBlob) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("view", true)}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -3944,7 +3944,7 @@ TEST_P(BlobTableInteTest, TestBlobViewFieldWithMultipleUpstreamTables) { BlobUtils::ToArrowField("view1", true), BlobUtils::ToArrowField("view2", true)}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -4222,7 +4222,7 @@ TEST_P(BlobTableInteTest, TestBlobViewWithFallbackPath) { BlobUtils::ToArrowField("blob", true)}; auto upstream_schema = arrow::schema(upstream_fields); std::map upstream_options = { - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -4269,7 +4269,7 @@ TEST_P(BlobTableInteTest, TestBlobViewWithFallbackPath) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("view", true)}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, diff --git a/test/inte/clean_inte_test.cpp b/test/inte/clean_inte_test.cpp index 89cadd4f7..6360b1138 100644 --- a/test/inte/clean_inte_test.cpp +++ b/test/inte/clean_inte_test.cpp @@ -190,7 +190,7 @@ class CleanInteTest : public testing::Test { }; TEST_F(CleanInteTest, TestExpireSnapshotFailover) { - std::string test_data_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/"; + std::string test_data_path = paimon::test::GetDataDir() + "/parquet/append_09.db/append_09/"; std::map clean_options = { {Options::MANIFEST_TARGET_FILE_SIZE, "8mb"}, {Options::FILE_SYSTEM, "local"}, @@ -218,7 +218,7 @@ TEST_F(CleanInteTest, TestExpireSnapshotFailover) { std::string table_path = dir->Str(); ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK(file_system_->Delete(PathUtil::JoinPath( - table_path, "manifest/manifest-list-616d1847-a02c-495f-9cca-2c8b7def0fec-1"))); + table_path, "manifest/manifest-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-1"))); CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); ASSERT_OK_AND_ASSIGN( std::unique_ptr commit_context, @@ -233,7 +233,7 @@ TEST_F(CleanInteTest, TestExpireSnapshotFailover) { std::string table_path = dir->Str(); ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK(file_system_->Delete(PathUtil::JoinPath( - table_path, "f1=10/bucket-1/data-10b9eea8-241d-4e4b-8ab8-2a82d72d79a2-0.orc"))); + table_path, "f1=10/bucket-1/data-7a912f84-04b7-4bbb-8dc6-53f4a292ea25-0.parquet"))); CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); ASSERT_OK_AND_ASSIGN( std::unique_ptr commit_context, @@ -272,7 +272,7 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshot) { ASSERT_TRUE(dir); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, "orc"}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::FILE_SYSTEM, "local"}, @@ -416,7 +416,7 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshotWithIOException) { auto schema = arrow::schema(arrow::FieldVector({string_field, int_field, int_field1, double_field})); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, "orc"}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::FILE_SYSTEM, "local"}, @@ -583,7 +583,7 @@ TEST_F(CleanInteTest, TestOrphanFilesClean) { ASSERT_TRUE(arrow::ExportSchema(*schema, &arrow_schema).ok()); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, "orc"}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::FILE_SYSTEM, "local"}, @@ -701,7 +701,7 @@ TEST_F(CleanInteTest, TestOrphanFilesCleanWithFileRetainCondition) { ASSERT_TRUE(arrow::ExportSchema(*schema, &arrow_schema).ok()); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, "orc"}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::FILE_SYSTEM, "local"}, @@ -817,7 +817,7 @@ TEST_F(CleanInteTest, TestOrphanFilesCleanWithIOException) { ASSERT_TRUE(dir); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, "orc"}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::FILE_SYSTEM, "local"}, diff --git a/test/inte/data_evolution_table_test.cpp b/test/inte/data_evolution_table_test.cpp index 303c6d700..00b42b274 100644 --- a/test/inte/data_evolution_table_test.cpp +++ b/test/inte/data_evolution_table_test.cpp @@ -79,7 +79,7 @@ class DataEvolutionTableTest : public ::testing::Test, } void CreateTable(const std::vector& partition_keys) const { - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, FileFormat()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -160,7 +160,7 @@ class DataEvolutionTableTest : public ::testing::Test, std::map CreateDataEvolutionTable( bool deletion_vectors_enabled, const std::map& extra_options = {}) const { - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, FileFormat()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -864,7 +864,7 @@ TEST_P(DataEvolutionTableTest, TestMultipleSharedShreddingMapsPartialOverwrite) arrow::field("map2", map_type), }; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, FileFormat()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -1192,7 +1192,7 @@ TEST_P(DataEvolutionTableTest, TestMoreData) { TEST_P(DataEvolutionTableTest, TestOnlyRowTrackingEnabled) { std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, FileFormat()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -1236,7 +1236,7 @@ TEST_P(DataEvolutionTableTest, TestExternalPath) { std::string external_test_dir = external_dir->Str(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, FileFormat()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -1488,7 +1488,7 @@ TEST_P(DataEvolutionTableTest, TestPartitionWithPredicate) { } std::vector partition_keys = {"f1"}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, FileFormat()}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, FileFormat()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, {"parquet.write.max-row-group-length", "1"}}; if (file_format == "mosaic") { @@ -2136,7 +2136,7 @@ TEST_P(DataEvolutionTableTest, TestFormatPredicatePushDownWithoutFileIndex) { std::map options = {{Options::FILE_INDEX_READ_ENABLED, "false"}, {Options::WRITE_BATCH_SIZE, "1"}, {"parquet.page.size", "1"}, - {"parquet.enable-dictionary", "false"}, + {"parquet.enable.dictionary", "false"}, {"parquet.write.enable-page-index", "true"}, {"parquet.read.enable-page-index-filter", "true"}, {"orc.stripe.size", "1"}, @@ -2187,7 +2187,7 @@ TEST_P(DataEvolutionTableTest, TestPredicate) { return; } if (FileFormat() == "mosaic") { - CreateTable(/*partition_keys=*/{}, {{Options::MANIFEST_FORMAT, "orc"}, + CreateTable(/*partition_keys=*/{}, {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, FileFormat()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -2331,7 +2331,7 @@ TEST_P(DataEvolutionTableTest, TestIOException) { } TEST_P(DataEvolutionTableTest, TestWithRowIds) { - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, FileFormat()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, diff --git a/test/inte/global_index_test.cpp b/test/inte/global_index_test.cpp index 84ab22df8..f42934782 100644 --- a/test/inte/global_index_test.cpp +++ b/test/inte/global_index_test.cpp @@ -84,7 +84,7 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter } void CreateTable(const std::vector& partition_keys) const { - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format_}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -259,7 +259,7 @@ TEST_P(GlobalIndexTest, TestWriteLuminaIndex) { {"lumina.encoding.type", "rawf32"}, {"lumina.search.parallel_number", "10"}}; - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format_}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -318,7 +318,7 @@ TEST_P(GlobalIndexTest, TestWriteLuminaIndexWithMismatchedDimension) { {"lumina.search.parallel_number", "10"}}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format_}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format_}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, {Options::READ_BATCH_SIZE, "1"}}; @@ -361,7 +361,7 @@ TEST_P(GlobalIndexTest, TestWriteAndQueryLuminaIndexWithOrcDictionaryStringTags) R"([{"key_name":"color","type":"enum","value_type":"string"},)" R"({"key_name":"labels","type":"enum","value_type":"string"}])"}}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format_}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format_}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, {"orc.dictionary-key-size-threshold", "1"}, {"orc.read.enable-lazy-decoding", "true"}}; @@ -956,7 +956,7 @@ TEST_P(GlobalIndexTest, TestWriteCommitScanReadIndexWithPartition) { {"lumina.encoding.type", "rawf32"}, {"lumina.search.parallel_number", "10"}}; auto schema = arrow::schema(fields); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format_}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -1097,7 +1097,7 @@ TEST_P(GlobalIndexTest, TestWriteCommitScanReadIndexWithScore) { {"lumina.encoding.type", "rawf32"}, {"lumina.search.parallel_number", "10"}}; auto schema = arrow::schema(fields); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format_}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -1243,7 +1243,7 @@ TEST_P(GlobalIndexTest, TestWriteAndQueryLuminaIndexWithTagNullAndEmptyValues) { R"({"key_name":"scores","type":"range","value_type":"float"},)" R"({"key_name":"category","type":"enum","value_type":"int32"},)" R"({"key_name":"category_ids","type":"enum","value_type":"int32"}])"}}; - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format_}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -1851,7 +1851,7 @@ TEST_P(GlobalIndexTest, TestScanIndexWithTwoIndexes) { {"lumina.encoding.type", "rawf32"}, {"lumina.search.parallel_number", "10"}}; auto schema = arrow::schema(fields); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format_}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -1919,7 +1919,7 @@ TEST_P(GlobalIndexTest, TestDataEvolutionBatchScanWithExternalPath) { arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::list(arrow::float32())), arrow::field("f2", arrow::int32()), arrow::field("f3", arrow::float64())}; auto schema = arrow::schema(fields); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format_}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -1986,7 +1986,7 @@ TEST_P(GlobalIndexTest, TestIOException) { ])") .ValueOrDie(); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format_}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -2312,7 +2312,7 @@ TEST_P(GlobalIndexTest, TestLuceneWriteCommitScanReadIndexWithScore) { {"lucene-fts.write.omit-term-freq-and-position", "false"}, {"lucene-fts.write.tmp.directory", tmp_dir->Str()}}; auto schema = arrow::schema(fields); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format_}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -2398,7 +2398,7 @@ TEST_P(GlobalIndexTest, TestWriteCommitScanReadLuceneIndexWithPartition) { {"lucene-fts.write.omit-term-freq-and-position", "false"}, {"lucene-fts.write.tmp.directory", tmp_dir->Str()}}; auto schema = arrow::schema(fields); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format_}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -2761,7 +2761,7 @@ TEST_P(GlobalIndexTest, TestBTreeEmptyStringKeyPredicates) { TEST_P(GlobalIndexTest, TestBTreeWriteCommitScanReadIndexWithPartition) { // BTree index with partitioned table. Each partition's data is sorted by f0 independently. auto schema = arrow::schema(fields_); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format_}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -2906,7 +2906,7 @@ TEST_P(GlobalIndexTest, TestBTreeWithPartitionAndCustomExecutor) { // Test that UnionGlobalIndexReader uses a custom 8-thread executor to read // btree indexes from two partitions in parallel. auto schema = arrow::schema(fields_); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format_}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -3327,7 +3327,7 @@ TEST_P(GlobalIndexTest, TestBTreeWithLumina) { {"lumina.encoding.type", "rawf32"}, {"lumina.search.parallel_number", "10"}}; auto schema = arrow::schema(fields); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format_}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, diff --git a/test/inte/nested_column_pruning_inte_test.cpp b/test/inte/nested_column_pruning_inte_test.cpp index b77d6bda9..90ecc470b 100644 --- a/test/inte/nested_column_pruning_inte_test.cpp +++ b/test/inte/nested_column_pruning_inte_test.cpp @@ -1164,7 +1164,7 @@ TEST_P(NestedColumnPruningInteTest, NestedStructMapSelectedKeysWithPredicate) { {Options::BUCKET, "-1"}, {Options::WRITE_BATCH_SIZE, "1"}, {"parquet.page.size", "1"}, - {"parquet.enable-dictionary", "false"}, + {"parquet.enable.dictionary", "false"}, {"parquet.write.enable-page-index", "true"}, {"parquet.write.max-row-group-length", "1"}, {"parquet.read.enable-page-index-filter", "true"}, diff --git a/test/inte/pk_compaction_inte_test.cpp b/test/inte/pk_compaction_inte_test.cpp index 8e9150aee..f8015d6ee 100644 --- a/test/inte/pk_compaction_inte_test.cpp +++ b/test/inte/pk_compaction_inte_test.cpp @@ -539,7 +539,7 @@ TEST_P(PkCompactionInteTest, TestKeyValueTableDvCompactionWithMapSharedShredding {"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.tags.map.shared-shredding.max-columns", "2"}, {"parquet.page.size", "1"}, - {"parquet.enable-dictionary", "false"}, + {"parquet.enable.dictionary", "false"}, {"parquet.write.enable-page-index", "true"}, {"parquet.write.max-row-group-length", "1"}, {"parquet.read.enable-page-index-filter", "true"}, diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index 343d99605..68690c4d1 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -1016,7 +1016,7 @@ TEST(SystemTableReadInteTest, TestReadOptimizedPrimaryKeyProjectionAndPredicateP {Options::BUCKET_KEY, "k"}, {Options::WRITE_BATCH_SIZE, "1"}, {"parquet.page.size", "1"}, - {"parquet.enable-dictionary", "false"}, + {"parquet.enable.dictionary", "false"}, {"parquet.write.enable-page-index", "true"}, {"parquet.write.max-row-group-length", "1"}, {"parquet.read.enable-page-index-filter", "true"}}; @@ -1239,8 +1239,8 @@ TEST(SystemTableReadInteTest, TestReadFilesSystemTableForPartitionedTable) { }; auto schema = arrow::schema(fields); std::map options = {{Options::FILE_SYSTEM, "local"}, - {Options::FILE_FORMAT, "orc"}, - {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::BUCKET, "1"}}; auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -1363,8 +1363,8 @@ TEST(SystemTableReadInteTest, TestReadFilesSystemTableForDatePartition) { }; auto schema = arrow::schema(fields); std::map options = {{Options::FILE_SYSTEM, "local"}, - {Options::FILE_FORMAT, "orc"}, - {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::BUCKET, "1"}, {Options::BUCKET_KEY, "v"}}; auto dir = UniqueTestDirectory::Create(); @@ -4284,8 +4284,8 @@ TEST(SystemTableReadInteTest, TestReadGlobalAllTableOptions) { TEST(SystemTableReadInteTest, TestReadGlobalTables) { std::map options = {{Options::FILE_SYSTEM, "local"}, - {Options::FILE_FORMAT, "orc"}, - {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::BUCKET, "1"}, {"owner", "alice"}, {"createdAt", "1000"}, @@ -4402,8 +4402,8 @@ TEST(SystemTableReadInteTest, TestReadGlobalTables) { TEST(SystemTableReadInteTest, TestReadGlobalPartitions) { std::map options = {{Options::FILE_SYSTEM, "local"}, - {Options::FILE_FORMAT, "orc"}, - {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::BUCKET, "1"}, {Options::BUCKET_KEY, "v"}}; auto dir = UniqueTestDirectory::Create(); @@ -4489,8 +4489,8 @@ TEST(SystemTableReadInteTest, TestGlobalSystemTablesPropagateCorruptSchema) { TEST(SystemTableReadInteTest, TestPartitionsSystemTablePropagatesCorruptSnapshot) { std::map options = {{Options::FILE_SYSTEM, "local"}, - {Options::FILE_FORMAT, "orc"}, - {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::BUCKET, "1"}, {Options::BUCKET_KEY, "v"}}; auto dir = UniqueTestDirectory::Create(); @@ -4524,8 +4524,8 @@ TEST(SystemTableReadInteTest, TestPartitionsSystemTablePropagatesCorruptSnapshot TEST(SystemTableReadInteTest, TestPartitionsSystemTablePropagatesCorruptManifest) { std::map options = {{Options::FILE_SYSTEM, "local"}, - {Options::FILE_FORMAT, "orc"}, - {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::BUCKET, "1"}, {Options::BUCKET_KEY, "v"}}; auto dir = UniqueTestDirectory::Create(); diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp index 6298137ea..8892380cb 100644 --- a/test/inte/realtime_write_inte_test.cpp +++ b/test/inte/realtime_write_inte_test.cpp @@ -197,7 +197,7 @@ class RealtimeWriteInteTest : public ::testing::Test { arrow::field("pt", arrow::utf8())}; schema_ = arrow::schema(fields_); options_ = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, "orc"}, {Options::FILE_SYSTEM, "local"}, {Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}, {Options::TARGET_FILE_SIZE, "1048576"}, {Options::REALTIME_ENABLED, "true"}, @@ -1484,7 +1484,7 @@ TEST_F(RealtimeWriteInteTest, TestDiskPredicatePushdownWithoutMemoryFiltering) { options_[Options::FILE_FORMAT] = "parquet"; options_[Options::WRITE_BATCH_SIZE] = "1"; options_["parquet.page.size"] = "1"; - options_["parquet.enable-dictionary"] = "false"; + options_["parquet.enable.dictionary"] = "false"; options_["parquet.write.enable-page-index"] = "true"; options_["parquet.read.enable-page-index-filter"] = "true"; CreateTable(/*partition_keys=*/{}); @@ -1620,7 +1620,7 @@ TEST_F(RealtimeWriteInteTest, TestMemoryBatchStatisticsPredicatePushdownWithDisk options_[Options::WRITE_BATCH_SIZE] = "1"; options_[Options::REALTIME_STORE_STATS_MODE] = "full"; options_["parquet.page.size"] = "1"; - options_["parquet.enable-dictionary"] = "false"; + options_["parquet.enable.dictionary"] = "false"; options_["parquet.write.enable-page-index"] = "true"; options_["parquet.read.enable-page-index-filter"] = "true"; CreateTable(/*partition_keys=*/{}); @@ -1674,7 +1674,7 @@ TEST_F(RealtimeWriteInteTest, TestNullPredicateForMemoryAndDisk) { options_[Options::WRITE_BATCH_SIZE] = "1"; options_[Options::REALTIME_STORE_STATS_MODE] = "full"; options_["parquet.page.size"] = "1"; - options_["parquet.enable-dictionary"] = "false"; + options_["parquet.enable.dictionary"] = "false"; options_["parquet.write.enable-page-index"] = "true"; options_["parquet.write.max-row-group-length"] = "1"; options_["parquet.read.enable-page-index-filter"] = "true"; diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp index 11080c9d5..e8245e455 100644 --- a/test/inte/scan_and_read_inte_test.cpp +++ b/test/inte/scan_and_read_inte_test.cpp @@ -2153,7 +2153,7 @@ TEST_P(ScanAndReadInteTest, TestPkScanWithPostponeBucket) { arrow::field("_VALUE_KIND", arrow::int8())); auto schema = arrow::schema(fields); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-2"}, diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index 43ed0d1fb..6cce44236 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -2237,7 +2237,7 @@ TEST_P(WriteAndReadInteTest, TestPKWithParquetPageIndexFilter) { arrow::field("f2", arrow::int32()), arrow::field("f3", arrow::float64())}; auto schema = arrow::schema(fields); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, "parquet"}, {Options::TARGET_FILE_SIZE, "1048576"}, {Options::BUCKET, "1"}, @@ -2253,7 +2253,7 @@ TEST_P(WriteAndReadInteTest, TestPKWithParquetPageIndexFilter) { // filter is enabled below). {Options::WRITE_BATCH_SIZE, "1"}, {"parquet.page.size", "1"}, - {"parquet.enable-dictionary", "false"}, + {"parquet.enable.dictionary", "false"}, {"parquet.write.enable-page-index", "true"}, }; ASSERT_OK_AND_ASSIGN(auto helper, @@ -2352,7 +2352,7 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilter) { arrow::field("f2", arrow::int32()), arrow::field("f3", arrow::float64())}; auto schema = arrow::schema(fields); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, "parquet"}, {Options::TARGET_FILE_SIZE, "1048576"}, {Options::BUCKET, "-1"}, @@ -2363,7 +2363,7 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilter) { // without row-level filter the reader output is precisely that one row. {Options::WRITE_BATCH_SIZE, "1"}, {"parquet.page.size", "1"}, - {"parquet.enable-dictionary", "false"}, + {"parquet.enable.dictionary", "false"}, {"parquet.write.enable-page-index", "true"}, }; ASSERT_OK_AND_ASSIGN(auto helper, @@ -2459,7 +2459,7 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilterAndPrefetch) { arrow::field("f1", arrow::utf8())}; auto schema = arrow::schema(fields); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, "parquet"}, {Options::TARGET_FILE_SIZE, "1048576"}, {Options::BUCKET, "-1"}, @@ -2469,7 +2469,7 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilterAndPrefetch) { // in 4 row groups of 4 single-row pages. {Options::WRITE_BATCH_SIZE, "1"}, {"parquet.page.size", "1"}, - {"parquet.enable-dictionary", "false"}, + {"parquet.enable.dictionary", "false"}, {"parquet.write.enable-page-index", "true"}, {"parquet.write.max-row-group-length", "4"}, {"parquet.read.enable-page-index-filter", "true"}, @@ -2546,7 +2546,7 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetMetadataCache) { arrow::field("f1", arrow::int32())}; auto schema = arrow::schema(fields); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, "parquet"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, "parquet"}, {Options::TARGET_FILE_SIZE, "1048576"}, {Options::BUCKET, "-1"}, {Options::FILE_SYSTEM, "local"}, }; @@ -2892,7 +2892,7 @@ TEST_P(WriteAndReadInteTest, TestAppendMapSharedShreddingWithPredicate) { {Options::FILE_SYSTEM, file_system}, {Options::WRITE_BATCH_SIZE, "1"}, {"parquet.page.size", "1"}, - {"parquet.enable-dictionary", "false"}, + {"parquet.enable.dictionary", "false"}, {"parquet.write.enable-page-index", "true"}, {"parquet.write.max-row-group-length", "1"}, {"parquet.read.enable-page-index-filter", "true"}, diff --git a/test/inte/write_inte_test.cpp b/test/inte/write_inte_test.cpp index 5b0b8a025..1dc000500 100644 --- a/test/inte/write_inte_test.cpp +++ b/test/inte/write_inte_test.cpp @@ -409,7 +409,7 @@ TEST_P(WriteInteTest, TestAppendTableBatchWrite) { auto schema = arrow::schema(fields); auto file_format = GetParam(); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -514,7 +514,7 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithOneBucket) { auto file_format = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, {Options::BUCKET_KEY, "f5"}, {Options::FILE_SYSTEM, "local"}, }; @@ -668,7 +668,7 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithPartitionAndMultiBuckets) { std::vector partition_keys = {"f2", "f1"}; auto file_format = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "2"}, {Options::BUCKET_KEY, "f0"}, {Options::FILE_SYSTEM, "local"}, }; @@ -824,7 +824,7 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithComplexType) { auto file_format = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, {Options::BUCKET_KEY, "f5"}, {Options::FILE_SYSTEM, "local"}, }; @@ -974,7 +974,7 @@ TEST_P(WriteInteTest, TestPkTableStreamWrite) { std::vector partition_keys = {"f1"}; auto file_format = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "2"}, {Options::BUCKET_KEY, "f0"}, {Options::FILE_SYSTEM, "local"}, }; @@ -1250,7 +1250,7 @@ TEST_P(WriteInteTest, TestPkTableBatchWrite) { std::vector partition_keys = {"f1"}; auto file_format = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "2"}, {Options::BUCKET_KEY, "f0"}, {Options::FILE_SYSTEM, "local"}, }; @@ -1411,7 +1411,7 @@ TEST_P(WriteInteTest, TestPkTableWriteWithNoPartitionKey) { std::vector partition_keys = {}; auto file_format = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "2"}, {Options::BUCKET_KEY, "f0"}, {Options::FILE_SYSTEM, "local"}, }; @@ -1650,7 +1650,7 @@ TEST_P(WriteInteTest, TestPkTableWriteWithComplexType) { std::vector partition_keys = {}; auto file_format = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, {Options::BUCKET_KEY, "f5"}, {Options::FILE_SYSTEM, "local"}, }; @@ -1822,7 +1822,7 @@ TEST_P(WriteInteTest, TestPkTableForceLookup) { std::vector partition_keys = {}; auto file_format = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, {Options::BUCKET_KEY, "f0"}, {Options::FILE_SYSTEM, "local"}, {Options::FORCE_LOOKUP, "true"}, {Options::WRITE_ONLY, "true"}}; @@ -1883,7 +1883,7 @@ TEST_P(WriteInteTest, TestPkTableEnableDeletionVector) { std::vector primary_keys = {"f0", "f1"}; std::vector partition_keys = {}; auto file_format = GetParam(); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, @@ -1943,7 +1943,7 @@ TEST_P(WriteInteTest, TestPkTableWriteWithIOException) { std::vector primary_keys = {"f0", "f1"}; std::vector partition_keys = {"f1"}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "2"}, {Options::BUCKET_KEY, "f0"}, {Options::FILE_SYSTEM, "local"}, }; @@ -2235,7 +2235,7 @@ TEST_F(WriteInteTest, TestAppendTableWriteWithAlterTable) { arrow::field("e", arrow::int32()), }; std::map options = {{Options::FILE_FORMAT, "orc"}, - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::FILE_SYSTEM, "local"}}; ASSERT_OK_AND_ASSIGN(auto helper, @@ -2314,7 +2314,7 @@ TEST_F(WriteInteTest, TestPKTableWriteWithAlterTable) { arrow::field("v2", arrow::int32()), }; std::map options = {{Options::FILE_FORMAT, "orc"}, - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::FILE_SYSTEM, "local"}}; ASSERT_OK_AND_ASSIGN(auto helper, @@ -2409,7 +2409,7 @@ TEST_P(WriteInteTest, TestWriteAndCommitIOException) { auto file_format = GetParam(); std::map options = { {Options::FILE_FORMAT, file_format}, - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::FILE_SYSTEM, "local"}, {Options::BUCKET, "2"}, @@ -2513,7 +2513,7 @@ TEST_P(WriteInteTest, TestWriteWithFieldId) { ::ArrowSchema c_schema; ASSERT_TRUE(arrow::ExportType(*arrow_data_type, &c_schema).ok()); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::FILE_SYSTEM, "local"}, @@ -2744,7 +2744,7 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithExternalPath) { auto file_format = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, @@ -3177,7 +3177,7 @@ TEST_P(WriteInteTest, TestWriteWithIOException) { auto file_format = GetParam(); std::map options = { {Options::FILE_FORMAT, file_format}, - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::FILE_SYSTEM, "local"}, {Options::BUCKET, "2"}, @@ -3237,7 +3237,7 @@ TEST_P(WriteInteTest, TestCommitWithIOException) { auto file_format = GetParam(); std::map options = { {Options::FILE_FORMAT, file_format}, - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::FILE_SYSTEM, "local"}, {Options::BUCKET, "2"}, @@ -3309,7 +3309,7 @@ TEST_P(WriteInteTest, TestWriteMemoryUse) { auto file_format = GetParam(); std::map options = { {Options::FILE_FORMAT, file_format}, - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::FILE_SYSTEM, "local"}, {Options::BUCKET, "2"}, @@ -3375,7 +3375,7 @@ TEST_P(WriteInteTest, TestAppendTableWithAllNull) { auto file_format = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, {Options::FILE_SYSTEM, "local"}, }; @@ -3415,7 +3415,7 @@ TEST_P(WriteInteTest, TestPkTablePostponeBucket) { auto schema = arrow::schema(fields); std::vector primary_keys = {"f0", "f1"}; auto file_format = GetParam(); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-2"}, @@ -3510,7 +3510,7 @@ TEST_F(WriteInteTest, TestBranchWrite) { ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); std::map options = {{Options::FILE_FORMAT, "orc"}, - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_SYSTEM, "local"}}; WriteContextBuilder context_builder(table_path, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, @@ -3611,7 +3611,7 @@ TEST_P(WriteInteTest, TestDataEvolutionWrite) { auto file_format = GetParam(); auto dir = UniqueTestDirectory::Create(); std::map options = {{Options::FILE_FORMAT, file_format}, - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, {Options::FILE_SYSTEM, "local"}}; @@ -3803,7 +3803,7 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithBlobType) { auto file_format = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, {Options::FILE_SYSTEM, "local"}, {Options::BLOB_AS_DESCRIPTOR, "true"}}; @@ -3914,7 +3914,7 @@ TEST_P(WriteInteTest, TestAppendTableWithDateFieldAsPartitionField) { auto schema = arrow::schema(fields); auto file_format = GetParam(); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -4570,7 +4570,7 @@ TEST_P(WriteInteTest, TestPkSpillableWithIOException) { std::vector partition_keys = {"f1"}; auto file_format = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "2"}, {Options::BUCKET_KEY, "f0"}, {Options::FILE_SYSTEM, "local"}, {Options::WRITE_BUFFER_SIZE, "1"}, {Options::WRITE_BUFFER_SPILLABLE, "true"}, @@ -4681,7 +4681,7 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithMultipleBlobFields) { auto schema = arrow::schema(fields); auto file_format = GetParam(); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -4826,7 +4826,7 @@ TEST_P(WriteInteTest, TestRowTrackingPartitionGroupOnCommit) { auto file_format = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -4941,7 +4941,7 @@ TEST_P(WriteInteTest, TestRowTrackingPartitionGroupOnCommitDisabled) { auto file_format = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -5039,7 +5039,7 @@ TEST_P(WriteInteTest, TestMultipleBlobFieldsSplitByTargetSize) { auto file_format = GetParam(); // Set a very small blob target file size to force splitting - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + std::map options = {{Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, diff --git a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-ed03e3fd-3ff4-4d88-9d5d-683700f78967-0 b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-ed03e3fd-3ff4-4d88-9d5d-683700f78967-0 index 1a0ea1eba..427e68590 100644 Binary files a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-ed03e3fd-3ff4-4d88-9d5d-683700f78967-0 and b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-ed03e3fd-3ff4-4d88-9d5d-683700f78967-0 differ diff --git a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-f2299c3d-c3f1-400f-ad3d-124e3a342389-0 b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-f2299c3d-c3f1-400f-ad3d-124e3a342389-0 index 8f99c331a..d8ca5da5d 100644 Binary files a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-f2299c3d-c3f1-400f-ad3d-124e3a342389-0 and b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-f2299c3d-c3f1-400f-ad3d-124e3a342389-0 differ diff --git a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-12b37c17-d02f-4409-8993-ac01a56210bf-0 b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-12b37c17-d02f-4409-8993-ac01a56210bf-0 index eea939b22..f8d0e6d11 100644 Binary files a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-12b37c17-d02f-4409-8993-ac01a56210bf-0 and b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-12b37c17-d02f-4409-8993-ac01a56210bf-0 differ diff --git a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-12b37c17-d02f-4409-8993-ac01a56210bf-1 b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-12b37c17-d02f-4409-8993-ac01a56210bf-1 index fbf9cda34..d025ffe7d 100644 Binary files a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-12b37c17-d02f-4409-8993-ac01a56210bf-1 and b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-12b37c17-d02f-4409-8993-ac01a56210bf-1 differ diff --git a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-83964df9-8a98-4f91-a4e9-05f3e07be3f9-0 b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-83964df9-8a98-4f91-a4e9-05f3e07be3f9-0 index c7de35987..e4cfe141c 100644 Binary files a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-83964df9-8a98-4f91-a4e9-05f3e07be3f9-0 and b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-83964df9-8a98-4f91-a4e9-05f3e07be3f9-0 differ diff --git a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-83964df9-8a98-4f91-a4e9-05f3e07be3f9-1 b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-83964df9-8a98-4f91-a4e9-05f3e07be3f9-1 index eea939b22..d12b180cb 100644 Binary files a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-83964df9-8a98-4f91-a4e9-05f3e07be3f9-1 and b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-83964df9-8a98-4f91-a4e9-05f3e07be3f9-1 differ diff --git a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/schema/schema-0 b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/schema/schema-0 index a1bd3ded6..b631fc3a1 100644 --- a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/schema/schema-0 +++ b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/schema/schema-0 @@ -34,8 +34,8 @@ "partitionKeys" : [ "key0", "key1" ], "primaryKeys" : [ ], "options" : { - "manifest.format" : "orc", + "manifest.format" : "avro", "file.format" : "orc" }, "timeMillis" : 1730430023755 -} \ No newline at end of file +} diff --git a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/schema/schema-1 b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/schema/schema-1 index cb81068fa..83b8cc9e2 100644 --- a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/schema/schema-1 +++ b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/schema/schema-1 @@ -35,8 +35,8 @@ "partitionKeys" : [ "key0", "key1" ], "primaryKeys" : [ ], "options" : { - "manifest.format" : "orc", + "manifest.format" : "avro", "file.format" : "orc" }, "timeMillis" : 1730430297413 -} \ No newline at end of file +} diff --git a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/branch/branch-rt/schema/schema-0 b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/branch/branch-rt/schema/schema-0 index 55f747076..d9a092c87 100644 --- a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/branch/branch-rt/schema/schema-0 +++ b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/branch/branch-rt/schema/schema-0 @@ -18,8 +18,8 @@ "partitionKeys" : [ "dt" ], "primaryKeys" : [ ], "options" : { - "manifest.format" : "orc", + "manifest.format" : "avro", "file.format" : "orc" }, "timeMillis" : 1744885904315 -} \ No newline at end of file +} diff --git a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/branch/branch-rt/schema/schema-1 b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/branch/branch-rt/schema/schema-1 index 9e0b0819e..724aef673 100644 --- a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/branch/branch-rt/schema/schema-1 +++ b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/branch/branch-rt/schema/schema-1 @@ -19,8 +19,8 @@ "primaryKeys" : [ "dt", "name" ], "options" : { "bucket" : "2", - "manifest.format" : "orc", + "manifest.format" : "avro", "file.format" : "orc" }, "timeMillis" : 1744885904631 -} \ No newline at end of file +} diff --git a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-6090bcc3-a5a6-4c51-bc5e-7af6498e5b99-0 b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-6090bcc3-a5a6-4c51-bc5e-7af6498e5b99-0 index 3469a68b1..4120d9e07 100644 Binary files a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-6090bcc3-a5a6-4c51-bc5e-7af6498e5b99-0 and b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-6090bcc3-a5a6-4c51-bc5e-7af6498e5b99-0 differ diff --git a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-b204b605-41df-407f-85d9-722bef2754f4-0 b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-b204b605-41df-407f-85d9-722bef2754f4-0 index 6e7a8b14c..f7b10cd0b 100644 Binary files a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-b204b605-41df-407f-85d9-722bef2754f4-0 and b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-b204b605-41df-407f-85d9-722bef2754f4-0 differ diff --git a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-1c1bf05e-c1d0-4513-bc39-b2cb7e2d4973-0 b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-1c1bf05e-c1d0-4513-bc39-b2cb7e2d4973-0 index c9c696013..5a21b96d1 100644 Binary files a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-1c1bf05e-c1d0-4513-bc39-b2cb7e2d4973-0 and b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-1c1bf05e-c1d0-4513-bc39-b2cb7e2d4973-0 differ diff --git a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-1c1bf05e-c1d0-4513-bc39-b2cb7e2d4973-1 b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-1c1bf05e-c1d0-4513-bc39-b2cb7e2d4973-1 index 7c3e8f780..230e1e70d 100644 Binary files a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-1c1bf05e-c1d0-4513-bc39-b2cb7e2d4973-1 and b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-1c1bf05e-c1d0-4513-bc39-b2cb7e2d4973-1 differ diff --git a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-779da483-08f4-4271-8a62-d6576f56bc80-0 b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-779da483-08f4-4271-8a62-d6576f56bc80-0 index c9c696013..8b4e5a62e 100644 Binary files a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-779da483-08f4-4271-8a62-d6576f56bc80-0 and b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-779da483-08f4-4271-8a62-d6576f56bc80-0 differ diff --git a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-779da483-08f4-4271-8a62-d6576f56bc80-1 b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-779da483-08f4-4271-8a62-d6576f56bc80-1 index 2d8d64790..2070099f9 100644 Binary files a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-779da483-08f4-4271-8a62-d6576f56bc80-1 and b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-779da483-08f4-4271-8a62-d6576f56bc80-1 differ diff --git a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/schema/schema-0 b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/schema/schema-0 index 55f747076..d9a092c87 100644 --- a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/schema/schema-0 +++ b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/schema/schema-0 @@ -18,8 +18,8 @@ "partitionKeys" : [ "dt" ], "primaryKeys" : [ ], "options" : { - "manifest.format" : "orc", + "manifest.format" : "avro", "file.format" : "orc" }, "timeMillis" : 1744885904315 -} \ No newline at end of file +} diff --git a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/schema/schema-1 b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/schema/schema-1 index db5856dc5..499fccc5f 100644 --- a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/schema/schema-1 +++ b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/schema/schema-1 @@ -18,9 +18,9 @@ "partitionKeys" : [ "dt" ], "primaryKeys" : [ ], "options" : { - "manifest.format" : "orc", + "manifest.format" : "avro", "file.format" : "orc", "scan.fallback-branch" : "rt" }, "timeMillis" : 1744885904624 -} \ No newline at end of file +} diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-6f30e9a0-53e1-4030-bf82-1e9b1d62d43f-0 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-6f30e9a0-53e1-4030-bf82-1e9b1d62d43f-0 index e02a4bc7a..3da1ad5d1 100644 Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-6f30e9a0-53e1-4030-bf82-1e9b1d62d43f-0 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-6f30e9a0-53e1-4030-bf82-1e9b1d62d43f-0 differ diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-6f30e9a0-53e1-4030-bf82-1e9b1d62d43f-1 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-6f30e9a0-53e1-4030-bf82-1e9b1d62d43f-1 index 8a2b97205..5dcc67f76 100644 Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-6f30e9a0-53e1-4030-bf82-1e9b1d62d43f-1 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-6f30e9a0-53e1-4030-bf82-1e9b1d62d43f-1 differ diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-bebb1b9f-3785-400b-8f58-8e6902f5016e-0 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-bebb1b9f-3785-400b-8f58-8e6902f5016e-0 index 5eb55ba49..2897e9b71 100644 Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-bebb1b9f-3785-400b-8f58-8e6902f5016e-0 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-bebb1b9f-3785-400b-8f58-8e6902f5016e-0 differ diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-bebb1b9f-3785-400b-8f58-8e6902f5016e-1 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-bebb1b9f-3785-400b-8f58-8e6902f5016e-1 index 1518ddb1a..2ee0906d9 100644 Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-bebb1b9f-3785-400b-8f58-8e6902f5016e-1 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-bebb1b9f-3785-400b-8f58-8e6902f5016e-1 differ diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-0 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-0 index 1641b8d34..34248ccc8 100644 Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-0 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-0 differ diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-1 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-1 index daea579f8..ee59a3182 100644 Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-1 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-1 differ diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-2 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-2 index 42382569a..07723ec4b 100644 Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-2 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-2 differ diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-3 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-3 index a69ce5f00..2b31bc2b9 100644 Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-3 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-3 differ diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-0 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-0 index 04ebc0214..c6a9e75da 100644 Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-0 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-0 differ diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-1 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-1 index 65664835a..4b5975760 100644 Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-1 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-1 differ diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-2 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-2 index 65664835a..3b778efd0 100644 Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-2 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-2 differ diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-3 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-3 index 04ebc0214..995dcac94 100644 Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-3 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-3 differ diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-4 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-4 index 65664835a..0dbea5922 100644 Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-4 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-4 differ diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-5 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-5 index 0336fb8d6..a47a14d7e 100644 Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-5 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-5 differ diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/schema/schema-0 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/schema/schema-0 index 96dd5b130..bbc18eece 100644 --- a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/schema/schema-0 +++ b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/schema/schema-0 @@ -32,9 +32,9 @@ "fields.default-aggregate-function" : "min", "lookup.remote-file.level-threshold" : "1", "merge-engine" : "aggregation", - "manifest.format" : "orc", + "manifest.format" : "avro", "file.format" : "orc", "deletion-vectors.enabled" : "true" }, "timeMillis" : 1775650042347 -} \ No newline at end of file +} diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-167e8d7d-d3ea-4e3b-81d5-fda85c774c29-0 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-167e8d7d-d3ea-4e3b-81d5-fda85c774c29-0 index a95c97f7b..22f5a4a00 100644 Binary files a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-167e8d7d-d3ea-4e3b-81d5-fda85c774c29-0 and b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-167e8d7d-d3ea-4e3b-81d5-fda85c774c29-0 differ diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-167e8d7d-d3ea-4e3b-81d5-fda85c774c29-1 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-167e8d7d-d3ea-4e3b-81d5-fda85c774c29-1 index 295cfe0b0..25e07f911 100644 Binary files a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-167e8d7d-d3ea-4e3b-81d5-fda85c774c29-1 and b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-167e8d7d-d3ea-4e3b-81d5-fda85c774c29-1 differ diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-daf4fd15-fe55-4216-8668-bac0b6a93781-0 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-daf4fd15-fe55-4216-8668-bac0b6a93781-0 index 4ad23b28b..64916dfee 100644 Binary files a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-daf4fd15-fe55-4216-8668-bac0b6a93781-0 and b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-daf4fd15-fe55-4216-8668-bac0b6a93781-0 differ diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-0 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-0 index 68eff1597..76da03aa9 100644 Binary files a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-0 and b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-0 differ diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-1 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-1 index 2dddc4691..d6d5f0a0d 100644 Binary files a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-1 and b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-1 differ diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-2 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-2 index 2dddc4691..93e23c94e 100644 Binary files a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-2 and b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-2 differ diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-3 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-3 index 8a6e7ed47..587549334 100644 Binary files a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-3 and b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-3 differ diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-c62379f7-6e8b-4a39-af6b-1ec7baecf2e9-0 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-c62379f7-6e8b-4a39-af6b-1ec7baecf2e9-0 index d2c7c01e0..266ffa1a7 100644 Binary files a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-c62379f7-6e8b-4a39-af6b-1ec7baecf2e9-0 and b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-c62379f7-6e8b-4a39-af6b-1ec7baecf2e9-0 differ diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-c62379f7-6e8b-4a39-af6b-1ec7baecf2e9-1 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-c62379f7-6e8b-4a39-af6b-1ec7baecf2e9-1 index 65e68ab9f..3a0a5211b 100644 Binary files a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-c62379f7-6e8b-4a39-af6b-1ec7baecf2e9-1 and b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-c62379f7-6e8b-4a39-af6b-1ec7baecf2e9-1 differ diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-0 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-0 index 277d45f2f..82fd8df80 100644 --- a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-0 +++ b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-0 @@ -39,9 +39,9 @@ "primaryKeys" : [ "k0", "k1", "p0", "p1" ], "options" : { "bucket" : "1", - "manifest.format" : "orc", + "manifest.format" : "avro", "file.format" : "orc", "sequence.field" : "s0,s1" }, "timeMillis" : 1735120478343 -} \ No newline at end of file +} diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-1 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-1 index 0bf55dbfa..1d6e3cc8f 100644 --- a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-1 +++ b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-1 @@ -39,9 +39,9 @@ "primaryKeys" : [ "k0", "tmp", "p0", "p1" ], "options" : { "bucket" : "1", - "manifest.format" : "orc", + "manifest.format" : "avro", "file.format" : "orc", "sequence.field" : "s0,s1" }, "timeMillis" : 1735207491195 -} \ No newline at end of file +} diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-2 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-2 index 13db16689..bc157552e 100644 --- a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-2 +++ b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-2 @@ -39,9 +39,9 @@ "primaryKeys" : [ "k1", "tmp", "p0", "p1" ], "options" : { "bucket" : "1", - "manifest.format" : "orc", + "manifest.format" : "avro", "file.format" : "orc", "sequence.field" : "s0,s1" }, "timeMillis" : 1735207514342 -} \ No newline at end of file +} diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-3 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-3 index 8ff9465ab..e1037a153 100644 --- a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-3 +++ b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-3 @@ -39,9 +39,9 @@ "primaryKeys" : [ "k1", "k0", "p0", "p1" ], "options" : { "bucket" : "1", - "manifest.format" : "orc", + "manifest.format" : "avro", "file.format" : "orc", "sequence.field" : "s0,s1" }, "timeMillis" : 1735207576625 -} \ No newline at end of file +} diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-4 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-4 index a7e52509b..23f2e45cc 100644 --- a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-4 +++ b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-4 @@ -43,9 +43,9 @@ "primaryKeys" : [ "k1", "k0", "p0", "p1" ], "options" : { "bucket" : "1", - "manifest.format" : "orc", + "manifest.format" : "avro", "file.format" : "orc", "sequence.field" : "s0,s1" }, "timeMillis" : 1735207715028 -} \ No newline at end of file +} diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-7008d4c0-b1b0-4237-ac6d-845b887efc92-0 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-7008d4c0-b1b0-4237-ac6d-845b887efc92-0 index fb7dcc927..0713a2fc3 100644 Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-7008d4c0-b1b0-4237-ac6d-845b887efc92-0 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-7008d4c0-b1b0-4237-ac6d-845b887efc92-0 differ diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-7008d4c0-b1b0-4237-ac6d-845b887efc92-1 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-7008d4c0-b1b0-4237-ac6d-845b887efc92-1 index 135345f6a..0ff37d234 100644 Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-7008d4c0-b1b0-4237-ac6d-845b887efc92-1 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-7008d4c0-b1b0-4237-ac6d-845b887efc92-1 differ diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-97ddb443-4cd9-42e9-9871-4d51d4fa1b49-0 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-97ddb443-4cd9-42e9-9871-4d51d4fa1b49-0 index 0a5198776..1317128e2 100644 Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-97ddb443-4cd9-42e9-9871-4d51d4fa1b49-0 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-97ddb443-4cd9-42e9-9871-4d51d4fa1b49-0 differ diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-eb34bdd2-2c23-49af-b5aa-537a596d8fe3-0 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-eb34bdd2-2c23-49af-b5aa-537a596d8fe3-0 index cffc81ea3..15c7d9913 100644 Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-eb34bdd2-2c23-49af-b5aa-537a596d8fe3-0 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-eb34bdd2-2c23-49af-b5aa-537a596d8fe3-0 differ diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-ed67f349-7c6f-4f51-8723-274a060042d1-0 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-ed67f349-7c6f-4f51-8723-274a060042d1-0 index 0a6862a6f..22f88c68a 100644 Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-ed67f349-7c6f-4f51-8723-274a060042d1-0 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-ed67f349-7c6f-4f51-8723-274a060042d1-0 differ diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-2663284f-7bac-441a-bfa1-d11bb24af95d-0 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-2663284f-7bac-441a-bfa1-d11bb24af95d-0 index 0e0eb50ee..89a934cac 100644 Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-2663284f-7bac-441a-bfa1-d11bb24af95d-0 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-2663284f-7bac-441a-bfa1-d11bb24af95d-0 differ diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-2663284f-7bac-441a-bfa1-d11bb24af95d-1 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-2663284f-7bac-441a-bfa1-d11bb24af95d-1 index 22f85d5d3..65ae80123 100644 Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-2663284f-7bac-441a-bfa1-d11bb24af95d-1 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-2663284f-7bac-441a-bfa1-d11bb24af95d-1 differ diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-0 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-0 index c7de35987..722a9a06a 100644 Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-0 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-0 differ diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-1 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-1 index 0e0eb50ee..2077b919d 100644 Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-1 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-1 differ diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-0 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-0 index 04d7af1a5..f60aee5da 100644 Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-0 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-0 differ diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-1 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-1 index a20428113..456e1eb2c 100644 Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-1 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-1 differ diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-2 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-2 index b07ea33d1..e8d006991 100644 Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-2 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-2 differ diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-3 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-3 index 7a0345807..15f30ebb7 100644 Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-3 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-3 differ diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-ba2c5ba9-2dfd-4d39-af49-9f88fa5c029f-0 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-ba2c5ba9-2dfd-4d39-af49-9f88fa5c029f-0 index 3023df2de..77010b30e 100644 Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-ba2c5ba9-2dfd-4d39-af49-9f88fa5c029f-0 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-ba2c5ba9-2dfd-4d39-af49-9f88fa5c029f-0 differ diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-ba2c5ba9-2dfd-4d39-af49-9f88fa5c029f-1 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-ba2c5ba9-2dfd-4d39-af49-9f88fa5c029f-1 index 41ad2bdf9..0097ed46f 100644 Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-ba2c5ba9-2dfd-4d39-af49-9f88fa5c029f-1 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-ba2c5ba9-2dfd-4d39-af49-9f88fa5c029f-1 differ diff --git a/test/test_data/parquet/append_09.db/append_09/schema/schema-0 b/test/test_data/parquet/append_09.db/append_09/schema/schema-0 index 2314054c7..b992da387 100644 --- a/test/test_data/parquet/append_09.db/append_09/schema/schema-0 +++ b/test/test_data/parquet/append_09.db/append_09/schema/schema-0 @@ -24,7 +24,7 @@ "options" : { "bucket" : "2", "bucket-key" : "f2", - "manifest.format" : "orc", + "manifest.format" : "avro", "file.format" : "parquet" }, "timeMillis" : 1755758260435