From 50d4207e439723ed1888a3b5d5846a1cf9a50967 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Wed, 19 Aug 2026 22:34:33 +0800
Subject: [PATCH 01/62] feat(realtime): add primary-key in-memory writes

Add typed primary-key store creation, an in-memory PK store, and a no-spill writer that materializes sealed mutations through MergeTreeWriter.

Keep writer-local compaction disabled, preserve sequence progress across sequential writer handoff, and reject unsupported V1 table options.
---
 .../realtime/arrow_realtime_store_factory.h   |   7 +-
 include/paimon/realtime/realtime_store.h      |  47 +-
 src/paimon/CMakeLists.txt                     |   5 +
 .../core/operation/file_store_write.cpp       |  25 +-
 .../operation/key_value_file_store_write.cpp  |  77 ++-
 .../operation/key_value_file_store_write.h    |   8 +
 .../key_value_file_store_write_test.cpp       |  48 ++
 .../realtime/arrow_realtime_store_factory.cpp |  55 +-
 .../realtime/primary_key_realtime_options.cpp |  58 ++
 .../realtime/primary_key_realtime_options.h   |  31 +
 .../primary_key_realtime_options_test.cpp     |  56 ++
 .../realtime/primary_key_realtime_store.cpp   | 563 ++++++++++++++++++
 .../realtime/primary_key_realtime_store.h     |  84 +++
 .../primary_key_realtime_store_test.cpp       | 244 ++++++++
 .../realtime/realtime_append_only_writer.cpp  |  11 +-
 .../core/realtime/realtime_context_impl.cpp   |  67 ++-
 .../core/realtime/realtime_context_impl.h     |  18 +-
 .../core/realtime/realtime_context_test.cpp   | 126 +---
 .../realtime/realtime_primary_key_writer.cpp  | 249 ++++++++
 .../realtime/realtime_primary_key_writer.h    |  89 +++
 20 files changed, 1690 insertions(+), 178 deletions(-)
 create mode 100644 src/paimon/core/realtime/primary_key_realtime_options.cpp
 create mode 100644 src/paimon/core/realtime/primary_key_realtime_options.h
 create mode 100644 src/paimon/core/realtime/primary_key_realtime_options_test.cpp
 create mode 100644 src/paimon/core/realtime/primary_key_realtime_store.cpp
 create mode 100644 src/paimon/core/realtime/primary_key_realtime_store.h
 create mode 100644 src/paimon/core/realtime/primary_key_realtime_store_test.cpp
 create mode 100644 src/paimon/core/realtime/realtime_primary_key_writer.cpp
 create mode 100644 src/paimon/core/realtime/realtime_primary_key_writer.h

diff --git a/include/paimon/realtime/arrow_realtime_store_factory.h b/include/paimon/realtime/arrow_realtime_store_factory.h
index 4d65743ab..da1b8de36 100644
--- a/include/paimon/realtime/arrow_realtime_store_factory.h
+++ b/include/paimon/realtime/arrow_realtime_store_factory.h
@@ -26,11 +26,8 @@ namespace paimon {
 /// Factory for Paimon's default Arrow-backed `RealtimeStore`.
 class PAIMON_EXPORT ArrowRealtimeStoreFactory : public RealtimeStoreFactory {
  public:
-    /// Creates an Arrow-backed store for one partition and bucket.
-    Result<std::shared_ptr<RealtimeStore>> Create(
-        std::unique_ptr<::ArrowSchema> write_schema, StatisticsMode statistics_mode,
-        const std::map<std::string, std::string>& options,
-        const std::shared_ptr<MemoryPool>& memory_pool) override;
+    /// Creates the built-in append or in-memory primary-key store.
+    Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override;
 };
 
 }  // namespace paimon
diff --git a/include/paimon/realtime/realtime_store.h b/include/paimon/realtime/realtime_store.h
index d02952acd..1e53c173e 100644
--- a/include/paimon/realtime/realtime_store.h
+++ b/include/paimon/realtime/realtime_store.h
@@ -25,8 +25,10 @@
 #include <optional>
 #include <string>
 #include <utility>
+#include <variant>
 #include <vector>
 
+#include "arrow/c/abi.h"
 #include "paimon/reader/batch_reader.h"
 #include "paimon/realtime/offset_range.h"
 #include "paimon/record_batch.h"
@@ -41,6 +43,31 @@ namespace paimon {
 class MemoryPool;
 class Predicate;
 
+struct PAIMON_EXPORT AppendRealtimeStoreCreateConfig {
+    StatisticsMode statistics_mode;
+};
+
+struct PAIMON_EXPORT PrimaryKeyRealtimeStoreCreateConfig {
+    std::vector<std::string> primary_keys;
+    /// Largest sequence restored from the committed snapshot. A PK store assigns one contiguous
+    /// sequence to every mutation in `Write` order, starting at the next value, and rejects
+    /// `Write` before the assigned sequence would exceed `INT64_MAX - 1`.
+    int64_t restore_max_sequence_number;
+};
+
+using RealtimeStoreCreateConfig =
+    std::variant<AppendRealtimeStoreCreateConfig, PrimaryKeyRealtimeStoreCreateConfig>;
+
+struct PAIMON_EXPORT RealtimeStoreCreateRequest {
+    /// Complete table write schema whose ownership is transferred to the factory.
+    std::unique_ptr<::ArrowSchema> write_schema;
+    std::map<std::string, std::string> options;
+    std::shared_ptr<MemoryPool> memory_pool;
+    std::map<std::string, std::string> partition;
+    int32_t bucket = -1;
+    RealtimeStoreCreateConfig mode_config;
+};
+
 /// A table record batch and its framework-assigned contiguous offset range.
 ///
 /// The batch contains only table write fields. Row `i` is associated with
@@ -133,8 +160,11 @@ class PAIMON_EXPORT RealtimeStore {
     ///
     /// Each output batch contains `_VALUE_KIND` first, followed by the fields requested by
     /// `context.read_schema` except a duplicate `_VALUE_KIND`. Concatenating all returned readers
-    /// must produce every matching row once. Paimon retains `view` for the lifetime of the
-    /// resulting framework reader.
+    /// must produce every matching row once. Primary-key readers additionally provide a non-null
+    /// `_SEQUENCE_NUMBER` when requested, are individually sorted by primary key, and contain at
+    /// most one mutation per key. Assigned sequences remain stable across views and queries;
+    /// readers need not be globally sorted with one another. Paimon retains `view` for the lifetime
+    /// of the resulting framework reader.
     virtual Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
         const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
         const RealtimeQueryContext& context) = 0;
@@ -157,16 +187,9 @@ class PAIMON_EXPORT RealtimeStoreFactory {
  public:
     virtual ~RealtimeStoreFactory() = default;
 
-    /// Creates a store configured with the supplied schema, statistics, options, and memory pool.
-    /// @param write_schema Complete table write schema whose ownership is transferred to the
-    /// factory. The factory may consume it or retain it in the created store.
-    /// @param statistics_mode Framework-parsed statistics collection mode.
-    /// @param options Effective table options available to the store.
-    /// @param memory_pool Memory pool provided by the write context.
-    virtual Result<std::shared_ptr<RealtimeStore>> Create(
-        std::unique_ptr<::ArrowSchema> write_schema, StatisticsMode statistics_mode,
-        const std::map<std::string, std::string>& options,
-        const std::shared_ptr<MemoryPool>& memory_pool) = 0;
+    /// Creates a store for the requested table mode and partition-bucket.
+    /// The factory consumes `request`, including ownership of `request.write_schema`.
+    virtual Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) = 0;
 };
 
 }  // namespace paimon
diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index 61fd7c96d..ea6159821 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -382,9 +382,12 @@ set(PAIMON_CORE_SRCS
     core/operation/write_restore.cpp
     core/realtime/arrow_realtime_store.cpp
     core/realtime/arrow_realtime_store_factory.cpp
+    core/realtime/primary_key_realtime_store.cpp
+    core/realtime/primary_key_realtime_options.cpp
     core/realtime/realtime_append_only_writer.cpp
     core/realtime/realtime_context.cpp
     core/realtime/realtime_context_impl.cpp
+    core/realtime/realtime_primary_key_writer.cpp
     core/postpone/postpone_bucket_writer.cpp
     core/schema/arrow_schema_validator.cpp
     core/schema/schema_manager.cpp
@@ -787,6 +790,8 @@ if(PAIMON_BUILD_TESTS)
                     core/manifest/index_manifest_file_handler_test.cpp
                     core/memory/writer_memory_manager_test.cpp
                     core/realtime/arrow_realtime_store_test.cpp
+                    core/realtime/primary_key_realtime_store_test.cpp
+                    core/realtime/primary_key_realtime_options_test.cpp
                     core/realtime/realtime_context_test.cpp
                     core/realtime/realtime_reader_test.cpp
                     core/mergetree/levels_test.cpp
diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp
index 6807ae35e..fb83c254c 100644
--- a/src/paimon/core/operation/file_store_write.cpp
+++ b/src/paimon/core/operation/file_store_write.cpp
@@ -36,6 +36,7 @@
 #include "paimon/core/operation/key_value_file_store_write.h"
 #include "paimon/core/options/merge_engine.h"
 #include "paimon/core/postpone/postpone_bucket_file_store_write.h"
+#include "paimon/core/realtime/primary_key_realtime_options.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
 #include "paimon/core/schema/schema_manager.h"
 #include "paimon/core/schema/table_schema.h"
@@ -197,7 +198,26 @@ Result<std::unique_ptr<FileStoreWrite>> FileStoreWrite::Create(std::unique_ptr<W
     } else {
         // pk table
         if (ctx->GetRealtimeContext()) {
-            return Status::Invalid("real-time write currently supports append tables only");
+            PAIMON_RETURN_NOT_OK(ValidatePrimaryKeyRealtimeOptions(options));
+            if (ignore_previous_files) {
+                return Status::NotImplemented(
+                    "PK realtime v1 requires restore from the latest snapshot");
+            }
+            if (!ctx->GetWriteSchema().empty()) {
+                return Status::NotImplemented(
+                    "PK realtime v1 does not support a custom write schema");
+            }
+            PAIMON_ASSIGN_OR_RAISE(std::optional<Snapshot> latest_snapshot,
+                                   snapshot_manager->LatestSnapshot());
+            if (latest_snapshot) {
+                PAIMON_ASSIGN_OR_RAISE(RealtimeOffsetMap realtime_committed_offsets,
+                                       RealtimeCommitProperties::ReadOffsets(
+                                           latest_snapshot, options.GetFileSystem()));
+                PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
+                                       RealtimeContextImpl::Cast(ctx->GetRealtimeContext()));
+                PAIMON_RETURN_NOT_OK(realtime_context_impl->AdvanceCommittedProgress(
+                    latest_snapshot->Id(), realtime_committed_offsets));
+            }
         }
         if (options.GetBucket() == BucketModeDefine::POSTPONE_BUCKET) {
             return PostponeBucketFileStoreWrite::Create(
@@ -253,7 +273,8 @@ Result<std::unique_ptr<FileStoreWrite>> FileStoreWrite::Create(std::unique_ptr<W
             ctx->GetRootPath(), schema, arrow_schema, partition_schema, dv_maintainer_factory,
             io_manager, key_comparator, sequence_fields_comparator, merge_function_wrapper, options,
             ignore_previous_files, ctx->IsStreamingMode(), ctx->IgnoreNumBucketCheck(),
-            ctx->EnableMultiThreadSpill(), ctx->GetExecutor(), ctx->GetMemoryPool());
+            ctx->EnableMultiThreadSpill(), ctx->GetRealtimeContext(), ctx->GetExecutor(),
+            ctx->GetMemoryPool());
     }
 }
 
diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp
index 08c5ea0c3..4456ee1c2 100644
--- a/src/paimon/core/operation/key_value_file_store_write.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write.cpp
@@ -18,21 +18,29 @@
 
 #include "paimon/core/operation/key_value_file_store_write.h"
 
+#include <limits>
+#include <optional>
 #include <vector>
 
+#include "arrow/c/bridge.h"
 #include "paimon/common/data/binary_row.h"
+#include "paimon/core/compact/noop_compact_manager.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/io/data_file_meta.h"
 #include "paimon/core/manifest/manifest_file.h"
 #include "paimon/core/manifest/manifest_list.h"
 #include "paimon/core/mergetree/levels.h"
 #include "paimon/core/mergetree/merge_tree_writer.h"
+#include "paimon/core/operation/commit/realtime_commit_properties.h"
 #include "paimon/core/operation/file_store_scan.h"
 #include "paimon/core/operation/key_value_file_store_scan.h"
+#include "paimon/core/realtime/realtime_context_impl.h"
+#include "paimon/core/realtime/realtime_primary_key_writer.h"
 #include "paimon/core/schema/table_schema.h"
 #include "paimon/core/utils/file_store_path_factory.h"
 #include "paimon/core/utils/primary_key_table_utils.h"
 #include "paimon/core/utils/snapshot_manager.h"
+#include "paimon/realtime/realtime_context.h"
 
 namespace arrow {
 class Schema;
@@ -60,6 +68,7 @@ KeyValueFileStoreWrite::KeyValueFileStoreWrite(
     const std::shared_ptr<MergeFunctionWrapper<KeyValue>>& merge_function_wrapper,
     const CoreOptions& options, bool ignore_previous_files, bool is_streaming_mode,
     bool ignore_num_bucket_check, bool enable_multi_thread_spill,
+    const std::shared_ptr<RealtimeContext>& realtime_context,
     const std::shared_ptr<Executor>& executor, const std::shared_ptr<MemoryPool>& pool)
     : AbstractFileStoreWrite(file_store_path_factory, snapshot_manager, schema_manager, commit_user,
                              root_path, table_schema, schema, /*write_schema=*/schema,
@@ -67,6 +76,7 @@ KeyValueFileStoreWrite::KeyValueFileStoreWrite(
                              ignore_previous_files, is_streaming_mode, ignore_num_bucket_check,
                              executor, pool),
       enable_multi_thread_spill_(enable_multi_thread_spill),
+      realtime_context_(realtime_context),
       key_comparator_(key_comparator),
       user_defined_seq_comparator_(user_defined_seq_comparator),
       merge_function_wrapper_(merge_function_wrapper),
@@ -74,7 +84,25 @@ KeyValueFileStoreWrite::KeyValueFileStoreWrite(
           options_, key_comparator_, user_defined_seq_comparator_, compaction_metrics_,
           table_schema_, schema_, schema_manager_, io_manager_, cache_manager_,
           file_store_path_factory_, root_path_, pool_)),
-      logger_(Logger::GetLogger("KeyValueFileStoreWrite")) {}
+      logger_(Logger::GetLogger("KeyValueFileStoreWrite")) {
+    if (realtime_context_) {
+        writer_memory_manager_ = std::make_unique<NoopWriterMemoryManager>();
+    }
+}
+
+Status KeyValueFileStoreWrite::RefreshCommittedSnapshot(int64_t snapshot_id) {
+    if (!realtime_context_) {
+        return Status::Invalid("refresh committed snapshot requires a real-time writer");
+    }
+    PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, snapshot_manager_->LoadSnapshot(snapshot_id));
+    PAIMON_ASSIGN_OR_RAISE(
+        RealtimeOffsetMap committed_offsets,
+        RealtimeCommitProperties::ReadOffsets(std::optional<Snapshot>(std::move(snapshot)),
+                                              options_.GetFileSystem()));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
+                           RealtimeContextImpl::Cast(realtime_context_));
+    return realtime_context_impl->AdvanceCommittedProgress(snapshot_id, committed_offsets);
+}
 
 Result<std::unique_ptr<FileStoreScan>> KeyValueFileStoreWrite::CreateFileStoreScan(
     const std::shared_ptr<ScanFilter>& scan_filter) const {
@@ -109,19 +137,48 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
     PAIMON_ASSIGN_OR_RAISE(
         std::shared_ptr<Levels> levels,
         Levels::Create(key_comparator_, restore_data_files, options_.GetNumLevels()));
-    auto compact_strategy = compact_manager_factory_->CreateCompactStrategy();
-    PAIMON_ASSIGN_OR_RAISE(
-        std::shared_ptr<CompactManager> compact_manager,
-        compact_manager_factory_->CreateCompactManager(partition, bucket, compact_strategy,
-                                                       compact_executor_, levels, dv_maintainer));
+    std::map<std::string, std::string> partition_map;
+    int64_t materialized_max_sequence_number = restore_max_seq_number;
+    if (realtime_context_) {
+        std::vector<std::pair<std::string, std::string>> partition_values;
+        PAIMON_ASSIGN_OR_RAISE(partition_values,
+                               file_store_path_factory_->GeneratePartitionVector(partition));
+        partition_map =
+            std::map<std::string, std::string>(partition_values.begin(), partition_values.end());
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
+                               RealtimeContextImpl::Cast(realtime_context_));
+        const RealtimePartitionBucket partition_bucket(partition_map, bucket);
+        materialized_max_sequence_number = realtime_context_impl->GetMaterializedMaxSequenceNumber(
+            partition_bucket, restore_max_seq_number);
+        if (materialized_max_sequence_number == std::numeric_limits<int64_t>::max()) {
+            return Status::Invalid("PK sequence number has reached INT64_MAX");
+        }
+    }
+    std::shared_ptr<CompactManager> compact_manager;
+    if (realtime_context_) {
+        compact_manager = std::make_shared<NoopCompactManager>();
+    } else {
+        auto compact_strategy = compact_manager_factory_->CreateCompactStrategy();
+        PAIMON_ASSIGN_OR_RAISE(compact_manager, compact_manager_factory_->CreateCompactManager(
+                                                    partition, bucket, compact_strategy,
+                                                    compact_executor_, levels, dv_maintainer));
+    }
 
     PAIMON_ASSIGN_OR_RAISE(
         std::shared_ptr<MergeTreeWriter> writer,
         MergeTreeWriter::Create(
-            restore_max_seq_number, trimmed_primary_keys, data_file_path_factory, key_comparator_,
-            user_defined_seq_comparator_, merge_function_wrapper_, table_schema_->Id(), schema_,
-            options_, compact_manager, io_manager_, enable_multi_thread_spill_, pool_));
-    return writer;
+            materialized_max_sequence_number, trimmed_primary_keys, data_file_path_factory,
+            key_comparator_, user_defined_seq_comparator_, merge_function_wrapper_,
+            table_schema_->Id(), schema_, options_, compact_manager,
+            realtime_context_ ? nullptr : io_manager_, enable_multi_thread_spill_, pool_));
+    if (!realtime_context_) {
+        return std::shared_ptr<BatchWriter>(std::move(writer));
+    }
+    auto c_write_schema = std::make_unique<ArrowSchema>();
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema_, c_write_schema.get()));
+    return RealtimePrimaryKeyWriter::Create(
+        partition_map, bucket, std::move(c_write_schema), trimmed_primary_keys, realtime_context_,
+        writer, options_.ToMap(), pool_, materialized_max_sequence_number);
 }
 
 Status KeyValueFileStoreWrite::Close() {
diff --git a/src/paimon/core/operation/key_value_file_store_write.h b/src/paimon/core/operation/key_value_file_store_write.h
index 14457590f..66c362f2e 100644
--- a/src/paimon/core/operation/key_value_file_store_write.h
+++ b/src/paimon/core/operation/key_value_file_store_write.h
@@ -45,6 +45,7 @@ class SnapshotManager;
 class SchemaManager;
 class TableSchema;
 class IOManager;
+class RealtimeContext;
 struct KeyValue;
 template <typename T>
 class MergeFunctionWrapper;
@@ -65,8 +66,10 @@ class KeyValueFileStoreWrite : public AbstractFileStoreWrite {
         const std::shared_ptr<MergeFunctionWrapper<KeyValue>>& merge_function_wrapper,
         const CoreOptions& options, bool ignore_previous_files, bool is_streaming_mode,
         bool ignore_num_bucket_check, bool enable_multi_thread_spill,
+        const std::shared_ptr<RealtimeContext>& realtime_context,
         const std::shared_ptr<Executor>& executor, const std::shared_ptr<MemoryPool>& pool);
 
+    Status RefreshCommittedSnapshot(int64_t snapshot_id) override;
     Status Close() override;
 
  private:
@@ -79,8 +82,13 @@ class KeyValueFileStoreWrite : public AbstractFileStoreWrite {
     Result<std::unique_ptr<FileStoreScan>> CreateFileStoreScan(
         const std::shared_ptr<ScanFilter>& filter) const override;
 
+    bool IsRealtimeWrite() const override {
+        return realtime_context_ != nullptr;
+    }
+
  private:
     bool enable_multi_thread_spill_;
+    std::shared_ptr<RealtimeContext> realtime_context_;
     std::shared_ptr<FieldsComparator> key_comparator_;
     std::shared_ptr<FieldsComparator> user_defined_seq_comparator_;
     std::shared_ptr<MergeFunctionWrapper<KeyValue>> merge_function_wrapper_;
diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp
index 35d938af7..45462ea6e 100644
--- a/src/paimon/core/operation/key_value_file_store_write_test.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp
@@ -53,6 +53,7 @@
 #include "paimon/format/reader_builder.h"
 #include "paimon/fs/local/local_file_system.h"
 #include "paimon/reader/file_batch_reader.h"
+#include "paimon/realtime/realtime_context.h"
 #include "paimon/record_batch.h"
 #include "paimon/status.h"
 #include "paimon/testing/utils/test_helper.h"
@@ -303,6 +304,53 @@ TEST_F(KeyValueFileStoreWriteTest, TestPrepareCommitShouldSucceedWhenLookupEnabl
     ASSERT_EQ(commit_messages.size(), 1);
 }
 
+TEST_F(KeyValueFileStoreWriteTest, TestRealtimeWrite) {
+    const std::map<std::string, std::string> options = {
+        {Options::BUCKET, "1"},
+        {Options::WRITE_BUFFER_SIZE, "1"},
+    };
+    const std::shared_ptr<arrow::Schema> schema = arrow::schema({
+        arrow::field("id", arrow::int64(), /*nullable=*/false),
+        arrow::field("value", arrow::utf8()),
+    });
+    std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    CreateTable(dir->Str(), schema, options);
+    const std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar");
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    WriteContextBuilder builder(table_path, "test");
+    builder.SetOptions(options)
+        .WithStreamingMode(true)
+        .WithRealtimeContext(realtime_context)
+        .WithTempDirectory(dir->Str());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> write_context, builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         FileStoreWrite::Create(std::move(write_context)));
+
+    ASSERT_OK(writer->Write(MakeBatch(schema, R"([
+        [1, "old"],
+        [2, "two"],
+        [1, "new"]
+    ])")));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progresses,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_EQ(1, progresses.size());
+    ASSERT_EQ(OffsetRange(0, 3), progresses[0].offset_range);
+    std::shared_ptr<CommitMessageImpl> commit_message =
+        std::dynamic_pointer_cast<CommitMessageImpl>(progresses[0].commit_message);
+    ASSERT_NE(nullptr, commit_message);
+    int64_t row_count = 0;
+    for (const std::shared_ptr<DataFileMeta>& file :
+         commit_message->GetNewFilesIncrement().NewFiles()) {
+        row_count += file->row_count;
+    }
+    ASSERT_EQ(2, row_count);
+    ASSERT_EQ(0, TestHelper::CountChannelFiles(dir->GetFileSystem(), dir->Str()));
+    ASSERT_OK(writer->Close());
+}
+
 TEST_F(KeyValueFileStoreWriteTest,
        TestPrepareCommitShouldSucceedWhenDefaultCompactRewriterPathEnabled) {
     ASSERT_OK_AND_ASSIGN(
diff --git a/src/paimon/core/realtime/arrow_realtime_store_factory.cpp b/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
index 1d7219c41..e6e22edfd 100644
--- a/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
+++ b/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
@@ -21,29 +21,66 @@
 
 #include "arrow/c/bridge.h"
 #include "arrow/c/helpers.h"
+#include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/fields_comparator.h"
 #include "paimon/common/utils/scope_guard.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
+#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
 #include "paimon/core/realtime/arrow_realtime_store.h"
+#include "paimon/core/realtime/primary_key_realtime_store.h"
 #include "paimon/macros.h"
 
 namespace paimon {
 
 Result<std::shared_ptr<RealtimeStore>> ArrowRealtimeStoreFactory::Create(
-    std::unique_ptr<ArrowSchema> write_schema, StatisticsMode statistics_mode,
-    const std::map<std::string, std::string>&, const std::shared_ptr<MemoryPool>& memory_pool) {
-    if (!write_schema || !write_schema->release) {
+    RealtimeStoreCreateRequest&& request) {
+    if (!request.write_schema || !request.write_schema->release) {
         return Status::Invalid("real-time store write schema is null");
     }
-    ScopeGuard schema_guard([schema = write_schema.get()]() { ArrowSchemaRelease(schema); });
-    if (!memory_pool) {
+    ScopeGuard schema_guard(
+        [schema = request.write_schema.get()]() { ArrowSchemaRelease(schema); });
+    if (!request.memory_pool) {
         return Status::Invalid("real-time store memory pool is null");
     }
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> imported_schema,
-                                      arrow::ImportSchema(write_schema.get()));
-    std::shared_ptr<arrow::MemoryPool> arrow_pool = GetArrowPool(memory_pool);
-    return std::make_shared<ArrowRealtimeStore>(imported_schema, statistics_mode, memory_pool,
-                                                arrow_pool);
+                                      arrow::ImportSchema(request.write_schema.get()));
+    if (std::holds_alternative<AppendRealtimeStoreCreateConfig>(request.mode_config)) {
+        const AppendRealtimeStoreCreateConfig& append_config =
+            std::get<AppendRealtimeStoreCreateConfig>(request.mode_config);
+        std::shared_ptr<arrow::MemoryPool> arrow_pool = GetArrowPool(request.memory_pool);
+        return std::make_shared<ArrowRealtimeStore>(imported_schema, append_config.statistics_mode,
+                                                    request.memory_pool, arrow_pool);
+    }
+
+    const PrimaryKeyRealtimeStoreCreateConfig& primary_key_config =
+        std::get<PrimaryKeyRealtimeStoreCreateConfig>(request.mode_config);
+    std::vector<DataField> key_fields;
+    key_fields.reserve(primary_key_config.primary_keys.size());
+    for (const std::string& primary_key : primary_key_config.primary_keys) {
+        const int32_t field_index = imported_schema->GetFieldIndex(primary_key);
+        if (field_index < 0) {
+            return Status::Invalid("primary key ", primary_key, " is missing from write schema");
+        }
+        key_fields.emplace_back(field_index, imported_schema->field(field_index));
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FieldsComparator> key_comparator,
+                           FieldsComparator::Create(key_fields, /*is_ascending_order=*/true));
+    auto merge_function_wrapper_factory = []() {
+        auto merge_function = std::make_unique<DeduplicateMergeFunction>(
+            /*ignore_delete=*/false);
+        return std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge_function));
+    };
+    PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(request.options));
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<PrimaryKeyRealtimeStore> store,
+        PrimaryKeyRealtimeStore::Create(imported_schema, primary_key_config.primary_keys,
+                                        key_comparator, merge_function_wrapper_factory,
+                                        primary_key_config.restore_max_sequence_number,
+                                        core_options.GetReadBatchSize(), request.memory_pool));
+    return std::shared_ptr<RealtimeStore>(std::move(store));
 }
 
 }  // namespace paimon
diff --git a/src/paimon/core/realtime/primary_key_realtime_options.cpp b/src/paimon/core/realtime/primary_key_realtime_options.cpp
new file mode 100644
index 000000000..e9779a59e
--- /dev/null
+++ b/src/paimon/core/realtime/primary_key_realtime_options.cpp
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/realtime/primary_key_realtime_options.h"
+
+#include "paimon/core/core_options.h"
+
+namespace paimon {
+
+Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options) {
+    if (options.GetBucket() <= 0) {
+        return Status::NotImplemented("PK realtime v1 requires fixed buckets");
+    }
+    if (options.GetMergeEngine() != MergeEngine::DEDUPLICATE) {
+        return Status::NotImplemented("PK realtime v1 supports only the DEDUPLICATE merge engine");
+    }
+    if (options.DataEvolutionEnabled()) {
+        return Status::NotImplemented("PK realtime v1 does not support data evolution");
+    }
+    if (!options.GetFieldsSequenceGroups().empty()) {
+        return Status::NotImplemented("PK realtime v1 does not support sequence groups");
+    }
+    if (options.IgnoreDelete() || options.PartialUpdateRemoveRecordOnDelete() ||
+        options.AggregationRemoveRecordOnDelete() ||
+        !options.GetPartialUpdateRemoveRecordOnSequenceGroup().empty()) {
+        return Status::NotImplemented("PK realtime v1 requires default delete behavior");
+    }
+    if (!options.GetSequenceField().empty()) {
+        return Status::NotImplemented("PK realtime v1 does not support sequence.field");
+    }
+    if (!options.SequenceFieldSortOrderIsAscending()) {
+        return Status::NotImplemented(
+            "PK realtime v1 supports only ascending sequence.field.sort-order");
+    }
+    if (options.NeedLookup() || options.DeletionVectorsEnabled() ||
+        options.GetChangelogProducer() != ChangelogProducer::NONE) {
+        return Status::NotImplemented("PK realtime v1 does not support lookup or early MOR");
+    }
+    return Status::OK();
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/realtime/primary_key_realtime_options.h b/src/paimon/core/realtime/primary_key_realtime_options.h
new file mode 100644
index 000000000..a16d35778
--- /dev/null
+++ b/src/paimon/core/realtime/primary_key_realtime_options.h
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include "paimon/status.h"
+
+namespace paimon {
+
+class CoreOptions;
+
+/// Validates the table options supported by the in-memory PK realtime V1 path.
+Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options);
+
+}  // namespace paimon
diff --git a/src/paimon/core/realtime/primary_key_realtime_options_test.cpp b/src/paimon/core/realtime/primary_key_realtime_options_test.cpp
new file mode 100644
index 000000000..5d3ea7f67
--- /dev/null
+++ b/src/paimon/core/realtime/primary_key_realtime_options_test.cpp
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/realtime/primary_key_realtime_options.h"
+
+#include <map>
+#include <string>
+#include <vector>
+
+#include "paimon/core/core_options.h"
+#include "paimon/defs.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+TEST(PrimaryKeyRealtimeOptionsTest, TestSupportedOptions) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
+    ASSERT_OK(ValidatePrimaryKeyRealtimeOptions(options));
+}
+
+TEST(PrimaryKeyRealtimeOptionsTest, TestUnsupportedOptions) {
+    const std::string sequence_group =
+        std::string(Options::FIELDS_PREFIX) + ".value." + Options::SEQUENCE_GROUP;
+    const std::vector<std::map<std::string, std::string>> unsupported_options = {
+        {{Options::BUCKET, "0"}},
+        {{Options::BUCKET, "1"}, {Options::MERGE_ENGINE, "partial-update"}},
+        {{Options::BUCKET, "1"}, {Options::DATA_EVOLUTION_ENABLED, "true"}},
+        {{Options::BUCKET, "1"}, {sequence_group, "seq"}},
+        {{Options::BUCKET, "1"}, {Options::SEQUENCE_FIELD, "seq"}},
+        {{Options::BUCKET, "1"}, {Options::FORCE_LOOKUP, "true"}},
+        {{Options::BUCKET, "1"}, {Options::DELETION_VECTORS_ENABLED, "true"}},
+        {{Options::BUCKET, "1"}, {Options::CHANGELOG_PRODUCER, "input"}},
+    };
+    for (const std::map<std::string, std::string>& option_map : unsupported_options) {
+        ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
+        ASSERT_NOK(ValidatePrimaryKeyRealtimeOptions(options));
+    }
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
new file mode 100644
index 000000000..84afb97a4
--- /dev/null
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -0,0 +1,563 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/realtime/primary_key_realtime_store.h"
+
+#include <algorithm>
+#include <limits>
+#include <mutex>
+#include <utility>
+
+#include "arrow/api.h"
+#include "arrow/c/bridge.h"
+#include "paimon/common/data/binary_row_writer.h"
+#include "paimon/common/data/columnar/columnar_row_ref.h"
+#include "paimon/common/metrics/metrics_impl.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/common/types/row_kind.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/fields_comparator.h"
+#include "paimon/core/io/key_value_in_memory_record_reader.h"
+#include "paimon/core/io/key_value_projection_consumer.h"
+#include "paimon/core/io/key_value_projection_reader.h"
+#include "paimon/core/io/merged_key_value_record_reader.h"
+#include "paimon/core/key_value.h"
+#include "paimon/core/mergetree/compact/sort_merge_reader_with_loser_tree.h"
+#include "paimon/macros.h"
+
+namespace paimon {
+namespace {
+
+uint64_t GetArrayMemoryUsage(const std::shared_ptr<arrow::ArrayData>& data) {
+    uint64_t result = 0;
+    for (const std::shared_ptr<arrow::Buffer>& buffer : data->buffers) {
+        if (buffer) {
+            result += static_cast<uint64_t>(buffer->size());
+        }
+    }
+    for (const std::shared_ptr<arrow::ArrayData>& child : data->child_data) {
+        result += GetArrayMemoryUsage(child);
+    }
+    if (data->dictionary) {
+        result += GetArrayMemoryUsage(data->dictionary);
+    }
+    return result;
+}
+
+struct StoredBatch {
+    std::shared_ptr<arrow::StructArray> data;
+    std::vector<RecordBatch::RowKind> row_kinds;
+    OffsetRange offset_range;
+    int64_t first_sequence_number;
+    uint64_t memory_usage;
+};
+using BatchGroup = std::vector<std::shared_ptr<const StoredBatch>>;
+
+class Segment final : public RealtimeSegmentHandle {
+ public:
+    Segment(const OffsetRange& offset_range,
+            std::vector<std::shared_ptr<const StoredBatch>>&& batches)
+        : offset_range_(offset_range), batches_(std::move(batches)) {}
+
+    OffsetRange GetOffsetRange() const override {
+        return offset_range_;
+    }
+
+    const std::vector<std::shared_ptr<const StoredBatch>>& Batches() const {
+        return batches_;
+    }
+
+    uint64_t GetMemoryUsage() const {
+        uint64_t result = 0;
+        for (const std::shared_ptr<const StoredBatch>& batch : batches_) {
+            result += batch->memory_usage;
+        }
+        return result;
+    }
+
+ private:
+    OffsetRange offset_range_;
+    std::vector<std::shared_ptr<const StoredBatch>> batches_;
+};
+
+class PrimaryKeyRealtimeReadView final : public RealtimeReadView {
+ public:
+    explicit PrimaryKeyRealtimeReadView(std::vector<BatchGroup>&& groups)
+        : groups_(std::move(groups)) {
+        if (!groups_.empty()) {
+            offset_range_ = OffsetRange(groups_.front().front()->offset_range.begin,
+                                        groups_.back().back()->offset_range.end);
+        }
+    }
+
+    std::optional<OffsetRange> GetOffsetRange() const override {
+        return offset_range_;
+    }
+
+    const std::vector<BatchGroup>& Groups() const {
+        return groups_;
+    }
+
+ private:
+    std::vector<BatchGroup> groups_;
+    std::optional<OffsetRange> offset_range_;
+};
+
+class CommitBatchReader final : public BatchReader {
+ public:
+    CommitBatchReader(const std::shared_ptr<Segment>& segment,
+                      const std::shared_ptr<arrow::MemoryPool>& arrow_pool)
+        : segment_(segment), arrow_pool_(arrow_pool), metrics_(std::make_shared<MetricsImpl>()) {}
+
+    Result<ReadBatch> NextBatch() override {
+        if (!segment_ || next_batch_ >= static_cast<int32_t>(segment_->Batches().size())) {
+            return MakeEofBatch();
+        }
+        const std::shared_ptr<const StoredBatch>& stored = segment_->Batches()[next_batch_++];
+        const int64_t row_count = stored->data->length();
+        arrow::Int8Builder row_kind_builder(arrow_pool_.get());
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(row_kind_builder.Reserve(row_count));
+        if (stored->row_kinds.empty()) {
+            for (int64_t i = 0; i < row_count; ++i) {
+                row_kind_builder.UnsafeAppend(static_cast<int8_t>(RecordBatch::RowKind::INSERT));
+            }
+        } else {
+            for (RecordBatch::RowKind row_kind : stored->row_kinds) {
+                row_kind_builder.UnsafeAppend(static_cast<int8_t>(row_kind));
+            }
+        }
+        std::shared_ptr<arrow::Array> row_kind_array;
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(row_kind_builder.Finish(&row_kind_array));
+        arrow::ArrayVector arrays = {std::move(row_kind_array)};
+        arrays.insert(arrays.end(), stored->data->fields().begin(), stored->data->fields().end());
+        arrow::FieldVector fields = {
+            DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())};
+        const arrow::FieldVector& value_fields = stored->data->struct_type()->fields();
+        fields.insert(fields.end(), value_fields.begin(), value_fields.end());
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::StructArray> output,
+                                          arrow::StructArray::Make(arrays, fields));
+        auto c_array = std::make_unique<ArrowArray>();
+        auto c_schema = std::make_unique<ArrowSchema>();
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*output, c_array.get(), c_schema.get()));
+        return ReadBatch(std::move(c_array), std::move(c_schema));
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return metrics_;
+    }
+
+    void Close() override {
+        segment_.reset();
+    }
+
+ private:
+    std::shared_ptr<Segment> segment_;
+    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
+    std::shared_ptr<Metrics> metrics_;
+    int32_t next_batch_ = 0;
+};
+
+class KeyRangeBatchReader final : public BatchReader, public PrimaryKeyRangeProvider {
+ public:
+    KeyRangeBatchReader(std::unique_ptr<BatchReader>&& reader,
+                        const std::shared_ptr<InternalRow>& min_key,
+                        const std::shared_ptr<InternalRow>& max_key)
+        : reader_(std::move(reader)), min_key_(min_key), max_key_(max_key) {}
+
+    Result<ReadBatch> NextBatch() override {
+        return reader_->NextBatch();
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return reader_->GetReaderMetrics();
+    }
+
+    void Close() override {
+        reader_->Close();
+    }
+
+    std::shared_ptr<InternalRow> GetMinKey() const override {
+        return min_key_;
+    }
+
+    std::shared_ptr<InternalRow> GetMaxKey() const override {
+        return max_key_;
+    }
+
+ private:
+    std::unique_ptr<BatchReader> reader_;
+    std::shared_ptr<InternalRow> min_key_;
+    std::shared_ptr<InternalRow> max_key_;
+};
+
+}  // namespace
+
+class PrimaryKeyRealtimeStore::Impl {
+ public:
+    Impl(const std::shared_ptr<arrow::Schema>& write_schema, std::vector<std::string> primary_keys,
+         const std::shared_ptr<FieldsComparator>& key_comparator,
+         const std::function<std::shared_ptr<MergeFunctionWrapper<KeyValue>>()>&
+             merge_function_wrapper_factory,
+         int64_t next_sequence_number, int32_t read_batch_size,
+         const std::shared_ptr<MemoryPool>& memory_pool)
+        : write_schema_(write_schema),
+          primary_keys_(std::move(primary_keys)),
+          key_comparator_(key_comparator),
+          merge_function_wrapper_factory_(merge_function_wrapper_factory),
+          next_sequence_number_(next_sequence_number),
+          read_batch_size_(read_batch_size),
+          memory_pool_(memory_pool),
+          arrow_pool_(GetArrowPool(memory_pool)) {}
+
+    Result<std::shared_ptr<InternalRow>> CopyKey(const InternalRow& key) const {
+        auto result = std::make_shared<BinaryRow>(static_cast<int32_t>(primary_keys_.size()));
+        BinaryRowWriter writer(result.get(), /*initial_size=*/128, memory_pool_.get());
+        writer.Reset();
+        for (int32_t index = 0; index < static_cast<int32_t>(primary_keys_.size()); ++index) {
+            std::shared_ptr<arrow::Field> field =
+                write_schema_->GetFieldByName(primary_keys_[index]);
+            PAIMON_ASSIGN_OR_RAISE(InternalRow::FieldGetterFunc getter,
+                                   InternalRow::CreateFieldGetter(index, field->type(),
+                                                                  /*use_view=*/true));
+            PAIMON_ASSIGN_OR_RAISE(BinaryRowWriter::FieldSetterFunc setter,
+                                   BinaryRowWriter::CreateFieldSetter(index, field->type()));
+            setter(getter(key), &writer);
+        }
+        writer.Complete();
+        return std::static_pointer_cast<InternalRow>(result);
+    }
+
+    Result<std::pair<std::shared_ptr<InternalRow>, std::shared_ptr<InternalRow>>> GetKeyRange(
+        const std::shared_ptr<arrow::StructArray>& values) const {
+        arrow::ArrayVector key_arrays;
+        key_arrays.reserve(primary_keys_.size());
+        for (const std::string& primary_key : primary_keys_) {
+            std::shared_ptr<arrow::Array> key_array = values->GetFieldByName(primary_key);
+            if (!key_array) {
+                return Status::Invalid("primary key is missing from PK query batch: ", primary_key);
+            }
+            key_arrays.push_back(std::move(key_array));
+        }
+        auto context = std::make_shared<ColumnarBatchContext>(key_arrays, memory_pool_);
+        int64_t min_row = 0;
+        int64_t max_row = 0;
+        for (int64_t row = 1; row < values->length(); ++row) {
+            ColumnarRowRef current(context, row);
+            ColumnarRowRef min_key(context, min_row);
+            ColumnarRowRef max_key(context, max_row);
+            if (key_comparator_->CompareTo(current, min_key) < 0) {
+                min_row = row;
+            }
+            if (key_comparator_->CompareTo(current, max_key) > 0) {
+                max_row = row;
+            }
+        }
+        ColumnarRowRef min_key(context, min_row);
+        ColumnarRowRef max_key(context, max_row);
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InternalRow> copied_min, CopyKey(min_key));
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InternalRow> copied_max, CopyKey(max_key));
+        return std::make_pair(std::move(copied_min), std::move(copied_max));
+    }
+
+    Status Write(RealtimeWriteBatch&& write_batch) {
+        if (!write_batch.batch || !write_batch.batch->GetData()) {
+            return Status::Invalid("PK real-time write batch is null");
+        }
+        const int64_t row_count = write_batch.batch->GetData()->length;
+        if (row_count <= 0 || write_batch.offset_range.begin < 0 ||
+            write_batch.offset_range.Count() != row_count) {
+            return Status::Invalid("PK real-time offset range does not match batch row count");
+        }
+        const std::vector<RecordBatch::RowKind>& row_kinds = write_batch.batch->GetRowKind();
+        if (!row_kinds.empty() && static_cast<int64_t>(row_kinds.size()) != row_count) {
+            return Status::Invalid("PK real-time row-kind count does not match batch row count");
+        }
+        for (RecordBatch::RowKind row_kind : row_kinds) {
+            PAIMON_ASSIGN_OR_RAISE(const RowKind* validated,
+                                   RowKind::FromByteValue(static_cast<int8_t>(row_kind)));
+            static_cast<void>(validated);
+        }
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+            std::shared_ptr<arrow::Array> imported,
+            arrow::ImportArray(write_batch.batch->GetData(),
+                               arrow::struct_(write_schema_->fields())));
+        if (!imported || imported->type_id() != arrow::Type::STRUCT) {
+            return Status::Invalid("PK real-time write data is not a StructArray");
+        }
+        std::shared_ptr<arrow::StructArray> values =
+            checked_pointer_cast<arrow::StructArray>(imported);
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(values->ValidateFull());
+
+        std::lock_guard<std::mutex> lock(mutex_);
+        if (last_offset_ && write_batch.offset_range.begin != last_offset_.value()) {
+            return Status::Invalid("PK real-time offset ranges must be contiguous");
+        }
+        if (row_count > std::numeric_limits<int64_t>::max() - next_sequence_number_) {
+            return Status::Invalid("PK sequence range exceeds INT64_MAX");
+        }
+        auto stored = std::make_shared<const StoredBatch>(
+            StoredBatch{std::move(values), row_kinds, write_batch.offset_range,
+                        next_sequence_number_, GetArrayMemoryUsage(imported->data())});
+        building_batches_.push_back(std::move(stored));
+        building_memory_usage_ += building_batches_.back()->memory_usage;
+        last_offset_ = write_batch.offset_range.end;
+        next_sequence_number_ += row_count;
+        return Status::OK();
+    }
+
+    Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>> SealForCommit() {
+        std::lock_guard<std::mutex> lock(mutex_);
+        if (building_batches_.empty()) {
+            return std::optional<std::shared_ptr<RealtimeSegmentHandle>>();
+        }
+        const OffsetRange range(building_batches_.front()->offset_range.begin,
+                                building_batches_.back()->offset_range.end);
+        auto segment = std::make_shared<Segment>(range, std::move(building_batches_));
+        sealed_segments_.push_back(segment);
+        building_batches_.clear();
+        building_memory_usage_ = 0;
+        return std::optional<std::shared_ptr<RealtimeSegmentHandle>>(std::move(segment));
+    }
+
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
+        const std::shared_ptr<RealtimeSegmentHandle>& segment) {
+        std::shared_ptr<Segment> typed = std::dynamic_pointer_cast<Segment>(segment);
+        if (!typed) {
+            return Status::Invalid("segment was not created by the PK real-time store");
+        }
+        std::vector<std::unique_ptr<BatchReader>> result;
+        result.push_back(std::make_unique<CommitBatchReader>(typed, arrow_pool_));
+        return result;
+    }
+
+    Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() {
+        std::lock_guard<std::mutex> lock(mutex_);
+        std::vector<BatchGroup> groups;
+        groups.reserve(sealed_segments_.size() + (building_batches_.empty() ? 0 : 1));
+        for (const std::shared_ptr<Segment>& segment : sealed_segments_) {
+            groups.push_back(segment->Batches());
+        }
+        if (!building_batches_.empty()) {
+            groups.push_back(building_batches_);
+        }
+        return std::shared_ptr<RealtimeReadView>(new PrimaryKeyRealtimeReadView(std::move(groups)));
+    }
+
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
+        const std::shared_ptr<RealtimeReadView>& view, int64_t lower,
+        const RealtimeQueryContext& context) {
+        std::shared_ptr<PrimaryKeyRealtimeReadView> typed =
+            std::dynamic_pointer_cast<PrimaryKeyRealtimeReadView>(view);
+        if (!typed) {
+            return Status::Invalid("read view was not created by the PK real-time store");
+        }
+        if (!context.read_schema || !context.read_schema->release) {
+            return Status::Invalid("PK real-time query read schema is null");
+        }
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> requested,
+                                          arrow::ImportSchema(context.read_schema));
+        arrow::FieldVector output_fields = {
+            DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())};
+        std::vector<int32_t> projection = {KeyValueProjectionConsumer::kValueKindProjection};
+        for (const std::shared_ptr<arrow::Field>& field : requested->fields()) {
+            if (field->name() == SpecialFields::ValueKind().Name()) {
+                continue;
+            }
+            output_fields.push_back(field);
+            if (field->name() == SpecialFields::SequenceNumber().Name()) {
+                projection.push_back(KeyValueProjectionConsumer::kSequenceNumberProjection);
+                continue;
+            }
+            const int32_t index = write_schema_->GetFieldIndex(field->name());
+            if (index < 0) {
+                return Status::Invalid("PK real-time query field is missing from write schema: ",
+                                       field->name());
+            }
+            projection.push_back(index);
+        }
+
+        std::vector<std::unique_ptr<BatchReader>> result;
+        for (const BatchGroup& group : typed->Groups()) {
+            std::vector<std::unique_ptr<KeyValueRecordReader>> batch_readers;
+            std::shared_ptr<InternalRow> min_key;
+            std::shared_ptr<InternalRow> max_key;
+            for (const std::shared_ptr<const StoredBatch>& batch : group) {
+                if (batch->offset_range.end <= lower) {
+                    continue;
+                }
+                const int64_t offset = std::max<int64_t>(0, lower - batch->offset_range.begin);
+                const int64_t length = batch->data->length() - offset;
+                std::shared_ptr<arrow::Array> sliced = batch->data->Slice(offset, length);
+                std::shared_ptr<arrow::StructArray> selected =
+                    checked_pointer_cast<arrow::StructArray>(sliced);
+                using KeyRange =
+                    std::pair<std::shared_ptr<InternalRow>, std::shared_ptr<InternalRow>>;
+                PAIMON_ASSIGN_OR_RAISE(KeyRange key_range, GetKeyRange(selected));
+                if (!min_key || key_comparator_->CompareTo(*key_range.first, *min_key) < 0) {
+                    min_key = key_range.first;
+                }
+                if (!max_key || key_comparator_->CompareTo(*key_range.second, *max_key) > 0) {
+                    max_key = key_range.second;
+                }
+                std::vector<RecordBatch::RowKind> selected_kinds;
+                if (!batch->row_kinds.empty()) {
+                    selected_kinds.assign(batch->row_kinds.begin() + offset,
+                                          batch->row_kinds.end());
+                }
+                std::unique_ptr<KeyValueRecordReader> reader =
+                    std::make_unique<KeyValueInMemoryRecordReader>(
+                        batch->first_sequence_number + offset, selected, selected_kinds,
+                        primary_keys_, /*user_defined_sequence_fields=*/std::vector<std::string>(),
+                        /*sequence_fields_ascending=*/true, key_comparator_, memory_pool_);
+                std::shared_ptr<MergeFunctionWrapper<KeyValue>> batch_merge =
+                    merge_function_wrapper_factory_();
+                if (!batch_merge) {
+                    return Status::Invalid("merge function wrapper factory returned null");
+                }
+                batch_readers.push_back(std::make_unique<MergedKeyValueRecordReader>(
+                    std::move(reader), key_comparator_, batch_merge));
+            }
+            if (batch_readers.empty()) {
+                continue;
+            }
+            std::shared_ptr<MergeFunctionWrapper<KeyValue>> group_merge =
+                merge_function_wrapper_factory_();
+            if (!group_merge) {
+                return Status::Invalid("merge function wrapper factory returned null");
+            }
+            auto merged = std::make_unique<SortMergeReaderWithLoserTree>(
+                std::move(batch_readers), key_comparator_,
+                /*user_defined_seq_comparator=*/nullptr, group_merge);
+            PAIMON_ASSIGN_OR_RAISE(
+                std::unique_ptr<KeyValueProjectionReader> projected,
+                KeyValueProjectionReader::Create(std::move(merged), arrow::schema(output_fields),
+                                                 projection, read_batch_size_, memory_pool_));
+            result.push_back(
+                std::make_unique<KeyRangeBatchReader>(std::move(projected), min_key, max_key));
+        }
+        return result;
+    }
+
+    Status AdvanceCommittedOffset(int64_t committed_end_offset) {
+        std::lock_guard<std::mutex> lock(mutex_);
+        sealed_segments_.erase(
+            std::remove_if(sealed_segments_.begin(), sealed_segments_.end(),
+                           [committed_end_offset](const std::shared_ptr<Segment>& segment) {
+                               return segment->GetOffsetRange().end <= committed_end_offset;
+                           }),
+            sealed_segments_.end());
+        return Status::OK();
+    }
+
+    uint64_t GetMemoryUsage() const {
+        std::lock_guard<std::mutex> lock(mutex_);
+        uint64_t result = building_memory_usage_;
+        for (const std::shared_ptr<Segment>& segment : sealed_segments_) {
+            result += segment->GetMemoryUsage();
+        }
+        return result;
+    }
+
+ private:
+    std::shared_ptr<arrow::Schema> write_schema_;
+    std::vector<std::string> primary_keys_;
+    std::shared_ptr<FieldsComparator> key_comparator_;
+    std::function<std::shared_ptr<MergeFunctionWrapper<KeyValue>>()>
+        merge_function_wrapper_factory_;
+    int64_t next_sequence_number_;
+    int32_t read_batch_size_;
+    std::shared_ptr<MemoryPool> memory_pool_;
+    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
+    mutable std::mutex mutex_;
+    std::vector<std::shared_ptr<const StoredBatch>> building_batches_;
+    std::vector<std::shared_ptr<Segment>> sealed_segments_;
+    uint64_t building_memory_usage_ = 0;
+    std::optional<int64_t> last_offset_;
+};
+
+Result<std::shared_ptr<PrimaryKeyRealtimeStore>> PrimaryKeyRealtimeStore::Create(
+    const std::shared_ptr<arrow::Schema>& write_schema,
+    const std::vector<std::string>& primary_keys,
+    const std::shared_ptr<FieldsComparator>& key_comparator,
+    const std::function<std::shared_ptr<MergeFunctionWrapper<KeyValue>>()>&
+        merge_function_wrapper_factory,
+    int64_t restore_max_sequence_number, int32_t read_batch_size,
+    const std::shared_ptr<MemoryPool>& memory_pool) {
+    if (!write_schema || primary_keys.empty() || !key_comparator ||
+        !merge_function_wrapper_factory || !memory_pool || read_batch_size <= 0) {
+        return Status::Invalid("PK real-time store requires schema, keys, merge helpers, and pool");
+    }
+    if (restore_max_sequence_number < -1) {
+        return Status::Invalid("PK restore max sequence number must be at least -1");
+    }
+    if (restore_max_sequence_number == std::numeric_limits<int64_t>::max()) {
+        return Status::Invalid("PK sequence number has reached INT64_MAX");
+    }
+    for (const std::string& key : primary_keys) {
+        if (write_schema->GetFieldIndex(key) < 0) {
+            return Status::Invalid("primary key ", key, " is missing from write schema");
+        }
+    }
+    auto impl = std::make_unique<Impl>(
+        write_schema, primary_keys, key_comparator, merge_function_wrapper_factory,
+        restore_max_sequence_number + 1, read_batch_size, memory_pool);
+    return std::shared_ptr<PrimaryKeyRealtimeStore>(new PrimaryKeyRealtimeStore(std::move(impl)));
+}
+
+PrimaryKeyRealtimeStore::PrimaryKeyRealtimeStore(std::unique_ptr<Impl>&& impl)
+    : impl_(std::move(impl)) {}
+
+PrimaryKeyRealtimeStore::~PrimaryKeyRealtimeStore() = default;
+
+Status PrimaryKeyRealtimeStore::Write(RealtimeWriteBatch&& batch) {
+    return impl_->Write(std::move(batch));
+}
+
+Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>>
+PrimaryKeyRealtimeStore::SealForCommit() {
+    return impl_->SealForCommit();
+}
+
+Result<std::vector<std::unique_ptr<BatchReader>>> PrimaryKeyRealtimeStore::CreateCommitReaders(
+    const std::shared_ptr<RealtimeSegmentHandle>& segment) {
+    return impl_->CreateCommitReaders(segment);
+}
+
+Result<std::shared_ptr<RealtimeReadView>> PrimaryKeyRealtimeStore::AcquireReadView() {
+    return impl_->AcquireReadView();
+}
+
+Result<std::vector<std::unique_ptr<BatchReader>>> PrimaryKeyRealtimeStore::CreateQueryReaders(
+    const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
+    const RealtimeQueryContext& context) {
+    return impl_->CreateQueryReaders(view, offset_begin, context);
+}
+
+Status PrimaryKeyRealtimeStore::AdvanceCommittedOffset(int64_t committed_offset) {
+    return impl_->AdvanceCommittedOffset(committed_offset);
+}
+
+uint64_t PrimaryKeyRealtimeStore::GetMemoryUsage() const {
+    return impl_->GetMemoryUsage();
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.h b/src/paimon/core/realtime/primary_key_realtime_store.h
new file mode 100644
index 000000000..05225ed19
--- /dev/null
+++ b/src/paimon/core/realtime/primary_key_realtime_store.h
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "paimon/realtime/realtime_store.h"
+
+namespace arrow {
+class Schema;
+}  // namespace arrow
+
+namespace paimon {
+
+class FieldsComparator;
+struct KeyValue;
+class MemoryPool;
+class InternalRow;
+template <typename T>
+class MergeFunctionWrapper;
+
+/// Optional metadata exposed by PK query readers with a known inclusive key range.
+class PrimaryKeyRangeProvider {
+ public:
+    virtual ~PrimaryKeyRangeProvider() = default;
+
+    virtual std::shared_ptr<InternalRow> GetMinKey() const = 0;
+    virtual std::shared_ptr<InternalRow> GetMaxKey() const = 0;
+};
+
+/// In-memory store for primary-key real-time writes.
+class PrimaryKeyRealtimeStore final : public RealtimeStore {
+ public:
+    static Result<std::shared_ptr<PrimaryKeyRealtimeStore>> Create(
+        const std::shared_ptr<arrow::Schema>& write_schema,
+        const std::vector<std::string>& primary_keys,
+        const std::shared_ptr<FieldsComparator>& key_comparator,
+        const std::function<std::shared_ptr<MergeFunctionWrapper<KeyValue>>()>&
+            merge_function_wrapper_factory,
+        int64_t restore_max_sequence_number, int32_t read_batch_size,
+        const std::shared_ptr<MemoryPool>& memory_pool);
+
+    ~PrimaryKeyRealtimeStore() override;
+
+    Status Write(RealtimeWriteBatch&& batch) override;
+    Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>> SealForCommit() override;
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
+        const std::shared_ptr<RealtimeSegmentHandle>& segment) override;
+    Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override;
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
+        const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
+        const RealtimeQueryContext& context) override;
+    Status AdvanceCommittedOffset(int64_t committed_offset) override;
+    uint64_t GetMemoryUsage() const override;
+
+ private:
+    class Impl;
+    explicit PrimaryKeyRealtimeStore(std::unique_ptr<Impl>&& impl);
+
+    std::unique_ptr<Impl> impl_;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
new file mode 100644
index 000000000..9da272e0f
--- /dev/null
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/realtime/primary_key_realtime_store.h"
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "arrow/api.h"
+#include "arrow/c/bridge.h"
+#include "arrow/ipc/json_simple.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/fields_comparator.h"
+#include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
+#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+class PrimaryKeyRealtimeStoreTest : public testing::Test {
+ public:
+    void SetUp() override {
+        pool_ = std::shared_ptr<MemoryPool>(GetMemoryPool());
+        schema_ = arrow::schema(
+            {arrow::field("id", arrow::int64()), arrow::field("value", arrow::utf8())});
+        ASSERT_OK_AND_ASSIGN(key_comparator_,
+                             FieldsComparator::Create({DataField(0, schema_->field(0))},
+                                                      /*is_ascending_order=*/true));
+        auto merge_factory = []() {
+            auto merge_function =
+                std::make_unique<DeduplicateMergeFunction>(/*ignore_delete=*/false);
+            return std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge_function));
+        };
+        ASSERT_OK_AND_ASSIGN(
+            store_, PrimaryKeyRealtimeStore::Create(schema_, {"id"}, key_comparator_, merge_factory,
+                                                    /*restore_max_sequence_number=*/4,
+                                                    /*read_batch_size=*/1024, pool_));
+    }
+
+    std::unique_ptr<RecordBatch> MakeBatch(
+        const std::string& json, const std::vector<RecordBatch::RowKind>& row_kinds = {}) const {
+        std::shared_ptr<arrow::Array> array =
+            arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema_->fields()), json)
+                .ValueOrDie();
+        ArrowArray c_array;
+        EXPECT_TRUE(arrow::ExportArray(*array, &c_array).ok());
+        RecordBatchBuilder builder(&c_array);
+        builder.SetRowKinds(row_kinds);
+        return builder.Finish().value();
+    }
+
+    std::unique_ptr<ArrowSchema> MakeReadSchema(bool include_sequence) const {
+        arrow::FieldVector fields;
+        if (include_sequence) {
+            fields.push_back(
+                DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber()));
+        }
+        fields.insert(fields.end(), schema_->fields().begin(), schema_->fields().end());
+        auto c_schema = std::make_unique<ArrowSchema>();
+        EXPECT_TRUE(arrow::ExportSchema(*arrow::schema(fields), c_schema.get()).ok());
+        return c_schema;
+    }
+
+    void AssertReaderOutput(BatchReader* reader, const std::shared_ptr<arrow::DataType>& type,
+                            const std::string& json) const {
+        ASSERT_NE(nullptr, reader);
+        ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch());
+        ASSERT_FALSE(BatchReader::IsEofBatch(batch));
+        arrow::Result<std::shared_ptr<arrow::Array>> imported_result =
+            arrow::ImportArray(batch.first.get(), batch.second.get());
+        ASSERT_TRUE(imported_result.ok()) << imported_result.status().ToString();
+        std::shared_ptr<arrow::Array> actual = std::move(imported_result).ValueOrDie();
+        std::shared_ptr<arrow::Array> expected =
+            arrow::ipc::internal::json::ArrayFromJSON(type, json).ValueOrDie();
+        ASSERT_TRUE(actual->Equals(*expected))
+            << "expected: " << expected->ToString() << ", actual: " << actual->ToString();
+
+        ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch eof, reader->NextBatch());
+        ASSERT_TRUE(BatchReader::IsEofBatch(eof));
+        reader->Close();
+    }
+
+    std::shared_ptr<arrow::DataType> CommitType() const {
+        return arrow::struct_({
+            DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind()),
+            schema_->field(0),
+            schema_->field(1),
+        });
+    }
+
+    std::shared_ptr<arrow::DataType> QueryType() const {
+        return arrow::struct_({
+            DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind()),
+            DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber()),
+            schema_->field(0),
+            schema_->field(1),
+        });
+    }
+
+ protected:
+    std::shared_ptr<MemoryPool> pool_;
+    std::shared_ptr<arrow::Schema> schema_;
+    std::shared_ptr<FieldsComparator> key_comparator_;
+    std::shared_ptr<PrimaryKeyRealtimeStore> store_;
+};
+
+TEST_F(PrimaryKeyRealtimeStoreTest, TestWriteAndSeal) {
+    ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
+                         store_->SealForCommit());
+    ASSERT_FALSE(segment.has_value());
+    ASSERT_NOK_WITH_MSG(store_->Write(RealtimeWriteBatch{nullptr, OffsetRange(0, 0)}),
+                        "write batch is null");
+    ASSERT_NOK_WITH_MSG(
+        store_->Write(RealtimeWriteBatch{MakeBatch(R"([[1, "a"], [2, "b"]])"), OffsetRange(0, 0)}),
+        "offset range does not match batch row count");
+
+    ASSERT_OK(
+        store_->Write(RealtimeWriteBatch{MakeBatch(R"([[1, "a"], [2, "b"]])"), OffsetRange(0, 2)}));
+    ASSERT_NOK_WITH_MSG(
+        store_->Write(RealtimeWriteBatch{MakeBatch(R"([[4, "d"]])"), OffsetRange(3, 4)}),
+        "offset ranges must be contiguous");
+    ASSERT_OK(store_->Write(RealtimeWriteBatch{MakeBatch(R"([[3, "c"]])"), OffsetRange(2, 3)}));
+
+    ASSERT_OK_AND_ASSIGN(segment, store_->SealForCommit());
+    ASSERT_TRUE(segment.has_value());
+    ASSERT_EQ(OffsetRange(0, 3), segment.value()->GetOffsetRange());
+    ASSERT_GT(store_->GetMemoryUsage(), 0);
+
+    auto merge_factory = []() {
+        auto merge_function = std::make_unique<DeduplicateMergeFunction>(/*ignore_delete=*/false);
+        return std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge_function));
+    };
+    ASSERT_NOK_WITH_MSG(PrimaryKeyRealtimeStore::Create(
+                            schema_, {"id"}, key_comparator_, merge_factory,
+                            /*restore_max_sequence_number=*/-2, /*read_batch_size=*/1024, pool_),
+                        "restore max sequence number must be at least -1");
+}
+
+TEST_F(PrimaryKeyRealtimeStoreTest, TestCommitReaderPreservesMutations) {
+    ASSERT_OK(store_->Write(
+        RealtimeWriteBatch{MakeBatch(R"([[2, "old"], [1, "one"], [2, "new"]])",
+                                     {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::INSERT,
+                                      RecordBatch::RowKind::UPDATE_AFTER}),
+                           OffsetRange(0, 3)}));
+    ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
+                         store_->SealForCommit());
+    ASSERT_TRUE(segment.has_value());
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store_->CreateCommitReaders(segment.value()));
+    ASSERT_EQ(1, readers.size());
+    AssertReaderOutput(readers[0].get(), CommitType(),
+                       R"([[0, 2, "old"], [0, 1, "one"], [2, 2, "new"]])");
+}
+
+TEST_F(PrimaryKeyRealtimeStoreTest, TestMutationMerge) {
+    ASSERT_OK(store_->Write(
+        RealtimeWriteBatch{MakeBatch(R"([[1, "old"], [2, "two"]])"), OffsetRange(0, 2)}));
+    ASSERT_OK(store_->Write(RealtimeWriteBatch{
+        MakeBatch(R"([[1, "new"], [2, "gone"]])",
+                  {RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::DELETE}),
+        OffsetRange(2, 4)}));
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store_->AcquireReadView());
+    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(/*include_sequence=*/true);
+    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store_->CreateQueryReaders(view, /*offset_begin=*/0, context));
+    ASSERT_EQ(1, readers.size());
+    AssertReaderOutput(readers[0].get(), QueryType(), R"([[2, 7, 1, "new"], [3, 8, 2, "gone"]])");
+}
+
+TEST_F(PrimaryKeyRealtimeStoreTest, TestReadViewLifecycle) {
+    ASSERT_OK(store_->Write(RealtimeWriteBatch{MakeBatch(R"([[10, "a"], [11, "b"], [12, "c"]])"),
+                                               OffsetRange(10, 13)}));
+    ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
+                         store_->SealForCommit());
+    ASSERT_TRUE(segment.has_value());
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store_->AcquireReadView());
+    ASSERT_EQ(std::optional<OffsetRange>(OffsetRange(10, 13)), view->GetOffsetRange());
+
+    ASSERT_OK(store_->AdvanceCommittedOffset(13));
+    ASSERT_EQ(0, store_->GetMemoryUsage());
+    ASSERT_OK(
+        store_->Write(RealtimeWriteBatch{MakeBatch(R"([[13, "later"]])"), OffsetRange(13, 14)}));
+
+    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(/*include_sequence=*/true);
+    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store_->CreateQueryReaders(view, /*offset_begin=*/11, context));
+    ASSERT_EQ(1, readers.size());
+    AssertReaderOutput(readers[0].get(), QueryType(), R"([[0, 6, 11, "b"], [0, 7, 12, "c"]])");
+
+    std::unique_ptr<ArrowSchema> empty_schema = MakeReadSchema(/*include_sequence=*/true);
+    context.read_schema = empty_schema.get();
+    ASSERT_OK_AND_ASSIGN(readers, store_->CreateQueryReaders(view, /*offset_begin=*/13, context));
+    ASSERT_TRUE(readers.empty());
+}
+
+TEST_F(PrimaryKeyRealtimeStoreTest, TestQueryKeyRange) {
+    ASSERT_OK(store_->Write(
+        RealtimeWriteBatch{MakeBatch(R"([[5, "five"], [1, "one"]])"), OffsetRange(0, 2)}));
+    ASSERT_OK(store_->SealForCommit());
+    ASSERT_OK(store_->Write(
+        RealtimeWriteBatch{MakeBatch(R"([[9, "nine"], [7, "seven"]])"), OffsetRange(2, 4)}));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store_->AcquireReadView());
+    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(/*include_sequence=*/true);
+    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store_->CreateQueryReaders(view, /*offset_begin=*/0, context));
+    ASSERT_EQ(2, readers.size());
+    auto* first_range = dynamic_cast<PrimaryKeyRangeProvider*>(readers[0].get());
+    auto* second_range = dynamic_cast<PrimaryKeyRangeProvider*>(readers[1].get());
+    ASSERT_NE(nullptr, first_range);
+    ASSERT_NE(nullptr, second_range);
+    ASSERT_EQ(1, first_range->GetMinKey()->GetLong(0));
+    ASSERT_EQ(5, first_range->GetMaxKey()->GetLong(0));
+    ASSERT_EQ(7, second_range->GetMinKey()->GetLong(0));
+    ASSERT_EQ(9, second_range->GetMaxKey()->GetLong(0));
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/core/realtime/realtime_append_only_writer.cpp b/src/paimon/core/realtime/realtime_append_only_writer.cpp
index 9d519d791..21d6cfb74 100644
--- a/src/paimon/core/realtime/realtime_append_only_writer.cpp
+++ b/src/paimon/core/realtime/realtime_append_only_writer.cpp
@@ -47,7 +47,7 @@ Result<std::shared_ptr<RealtimeAppendOnlyWriter>> RealtimeAppendOnlyWriter::Crea
     std::unique_ptr<::ArrowSchema> write_schema,
     const std::shared_ptr<RealtimeContext>& realtime_context,
     const std::shared_ptr<AppendOnlyWriter>& file_writer,
-    const std::shared_ptr<arrow::Schema>& input_schema, StatisticsMode statistics_mode,
+    const std::shared_ptr<arrow::Schema>& input_schema,
     const std::map<std::string, std::string>& options,
     const std::shared_ptr<MemoryPool>& memory_pool) {
     if (!realtime_context) {
@@ -55,10 +55,11 @@ Result<std::shared_ptr<RealtimeAppendOnlyWriter>> RealtimeAppendOnlyWriter::Crea
     }
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
                            RealtimeContextImpl::Cast(realtime_context));
-    PAIMON_ASSIGN_OR_RAISE(
-        RealtimeStoreState store_state,
-        realtime_context_impl->GetOrCreateRealtimeStore(partition, bucket, std::move(write_schema),
-                                                        statistics_mode, options, memory_pool));
+    RealtimeStoreCreateRequest request{
+        std::move(write_schema),          options, memory_pool, partition, bucket,
+        AppendRealtimeStoreCreateConfig{statistics_mode}};
+    PAIMON_ASSIGN_OR_RAISE(RealtimeStoreState store_state,
+                           realtime_context_impl->GetOrCreateRealtimeStore(std::move(request)));
     return std::shared_ptr<RealtimeAppendOnlyWriter>(new RealtimeAppendOnlyWriter(
         store_state.store, file_writer, input_schema, store_state.initial_offset, memory_pool));
 }
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index f6bad5cf1..0a367b2cd 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -78,19 +78,16 @@ Status RealtimeContextImpl::Start() {
 }
 
 Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
-    const std::map<std::string, std::string>& partition, int32_t bucket,
-    std::unique_ptr<ArrowSchema> write_schema, StatisticsMode statistics_mode,
-    const std::map<std::string, std::string>& options,
-    const std::shared_ptr<MemoryPool>& memory_pool) {
+    RealtimeStoreCreateRequest&& request) {
     std::lock_guard<std::mutex> progress_lock(progress_mutex_);
     std::lock_guard<std::mutex> registry_lock(mutex_);
-    const RealtimePartitionBucket key(partition, bucket);
+    const RealtimePartitionBucket key(request.partition, request.bucket);
     int64_t initial_offset = 0;
     auto offset_iter = committed_offsets_.find(key);
     if (offset_iter != committed_offsets_.end()) {
         if (offset_iter->second == std::numeric_limits<int64_t>::max()) {
-            if (write_schema) {
-                ArrowSchemaRelease(write_schema.get());
+            if (request.write_schema) {
+                ArrowSchemaRelease(request.write_schema.get());
             }
             return Status::Invalid("real-time offset has reached INT64_MAX");
         }
@@ -98,8 +95,8 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     }
     auto iter = stores_.find(key);
     if (iter != stores_.end()) {
-        if (write_schema) {
-            ArrowSchemaRelease(write_schema.get());
+        if (request.write_schema) {
+            ArrowSchemaRelease(request.write_schema.get());
         }
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeReadView> read_view,
                                iter->second->AcquireReadView());
@@ -119,9 +116,8 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
         }
         return RealtimeStoreState{iter->second, initial_offset};
     }
-    PAIMON_ASSIGN_OR_RAISE(
-        std::shared_ptr<RealtimeStore> store,
-        factory_->Create(std::move(write_schema), statistics_mode, options, memory_pool));
+    Result<std::shared_ptr<RealtimeStore>> store_result = factory_->Create(std::move(request));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> store, std::move(store_result));
     stores_.emplace(key, store);
     if (offset_iter != committed_offsets_.end()) {
         reclaimed_offsets_.emplace(key, offset_iter->second);
@@ -129,6 +125,27 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     return RealtimeStoreState{std::move(store), initial_offset};
 }
 
+int64_t RealtimeContextImpl::GetMaterializedMaxSequenceNumber(
+    const RealtimePartitionBucket& partition_bucket, int64_t restored_max_sequence_number) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    auto [iter, inserted] =
+        materialized_max_sequence_numbers_.emplace(partition_bucket, restored_max_sequence_number);
+    if (!inserted && restored_max_sequence_number > iter->second) {
+        iter->second = restored_max_sequence_number;
+    }
+    return iter->second;
+}
+
+void RealtimeContextImpl::AdvanceMaterializedMaxSequenceNumber(
+    const RealtimePartitionBucket& partition_bucket, int64_t max_sequence_number) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    auto [iter, inserted] =
+        materialized_max_sequence_numbers_.emplace(partition_bucket, max_sequence_number);
+    if (!inserted && max_sequence_number > iter->second) {
+        iter->second = max_sequence_number;
+    }
+}
+
 Result<std::vector<RealtimePartitionBucketView>> RealtimeContextImpl::AcquireReadViews() {
     std::lock_guard<std::mutex> lock(mutex_);
     std::vector<RealtimePartitionBucketView> result;
@@ -230,28 +247,12 @@ Status RealtimeContextImpl::AdvanceCommittedProgress(int64_t snapshot_id,
             if (partition_bucket.bucket < 0 || committed_end_offset < 0) {
                 return Status::Invalid("invalid partition-bucket committed offset");
             }
-        }
-        // Only stores created by this context can contain state which cannot be restored in
-        // place. Offsets for other partition-buckets are reference state for lazy store creation
-        // and may be removed or rolled back without rebuilding the context.
-        std::lock_guard<std::mutex> registry_lock(mutex_);
-        for (const auto& store_entry : stores_) {
-            const RealtimePartitionBucket& partition_bucket = store_entry.first;
             auto previous_iter = committed_offsets_.find(partition_bucket);
-            if (previous_iter == committed_offsets_.end()) {
-                continue;
-            }
-
-            auto current_iter = committed_offsets.find(partition_bucket);
-            if (current_iter == committed_offsets.end()) {
-                return Status::Invalid(
-                    "real-time committed progress removed an active partition-bucket; recreate "
-                    "RealtimeContext");
-            }
-            if (current_iter->second < previous_iter->second) {
-                return Status::Invalid(
-                    "real-time committed offset moved backwards for an active partition-bucket; "
-                    "recreate RealtimeContext");
+            if (previous_iter != committed_offsets_.end()) {
+                if (committed_end_offset < previous_iter->second) {
+                    return Status::Invalid(
+                        "real-time partition-bucket committed offset cannot move backwards");
+                }
             }
         }
         committed_offsets_ = committed_offsets;
diff --git a/src/paimon/core/realtime/realtime_context_impl.h b/src/paimon/core/realtime/realtime_context_impl.h
index 66c324cab..45d07deeb 100644
--- a/src/paimon/core/realtime/realtime_context_impl.h
+++ b/src/paimon/core/realtime/realtime_context_impl.h
@@ -32,8 +32,8 @@
 #include <vector>
 
 #include "paimon/realtime/realtime_context.h"
+#include "paimon/realtime/realtime_store.h"
 #include "paimon/result.h"
-#include "paimon/statistics_mode.h"
 #include "paimon/visibility.h"
 
 struct ArrowSchema;
@@ -65,11 +65,13 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
     static Result<std::shared_ptr<RealtimeContextImpl>> Cast(
         const std::shared_ptr<RealtimeContext>& context);
 
-    Result<RealtimeStoreState> GetOrCreateRealtimeStore(
-        const std::map<std::string, std::string>& partition, int32_t bucket,
-        std::unique_ptr<::ArrowSchema> write_schema, StatisticsMode statistics_mode,
-        const std::map<std::string, std::string>& options,
-        const std::shared_ptr<MemoryPool>& memory_pool);
+    Result<RealtimeStoreState> GetOrCreateRealtimeStore(RealtimeStoreCreateRequest&& request);
+
+    int64_t GetMaterializedMaxSequenceNumber(const RealtimePartitionBucket& partition_bucket,
+                                             int64_t restored_max_sequence_number);
+
+    void AdvanceMaterializedMaxSequenceNumber(const RealtimePartitionBucket& partition_bucket,
+                                              int64_t max_sequence_number);
 
     Result<std::vector<RealtimePartitionBucketView>> AcquireReadViews();
 
@@ -79,9 +81,6 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
 
     Status ReleaseReadView(const std::string& opaque_ticket);
 
-    // Returns an error requiring a new context if a newer snapshot removes or moves committed
-    // progress backwards for a store created by this context. Progress for inactive stores is
-    // only reference state and can be replaced in place.
     Status AdvanceCommittedProgress(int64_t snapshot_id,
                                     const RealtimeOffsetMap& committed_offsets);
 
@@ -103,6 +102,7 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
     std::mutex mutex_;
     std::mutex progress_mutex_;
     std::map<RealtimePartitionBucket, std::shared_ptr<RealtimeStore>> stores_;
+    std::map<RealtimePartitionBucket, int64_t> materialized_max_sequence_numbers_;
     // Full-table progress used as the initial offset when a store is created lazily.
     RealtimeOffsetMap committed_offsets_;
     // Progress already reflected in stores owned by this context.
diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp
index 017820fd4..33701afac 100644
--- a/src/paimon/core/realtime/realtime_context_test.cpp
+++ b/src/paimon/core/realtime/realtime_context_test.cpp
@@ -91,14 +91,11 @@ class TestingRealtimeStore : public RealtimeStore {
 
 class TestingRealtimeStoreFactory : public RealtimeStoreFactory {
  public:
-    Result<std::shared_ptr<RealtimeStore>> Create(std::unique_ptr<ArrowSchema> write_schema,
-                                                  StatisticsMode,
-                                                  const std::map<std::string, std::string>&,
-                                                  const std::shared_ptr<MemoryPool>&) override {
-        if (!write_schema || !write_schema->release) {
+    Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
+        if (!request.write_schema || !request.write_schema->release) {
             return Status::Invalid("testing write schema is null");
         }
-        ArrowSchemaRelease(write_schema.get());
+        ArrowSchemaRelease(request.write_schema.get());
         auto store = std::make_shared<TestingRealtimeStore>();
         stores.push_back(store);
         return store;
@@ -122,20 +119,28 @@ Result<std::shared_ptr<RealtimeContextImpl>> CreateContext(
     return RealtimeContextImpl::Cast(context);
 }
 
-TEST(RealtimeContextTest, TestReusesStoreAndCapturesRegisteredViews) {
+Result<RealtimeStoreState> GetOrCreateAppendStore(
+    const std::shared_ptr<RealtimeContextImpl>& context,
+    const std::map<std::string, std::string>& partition, int32_t bucket,
+    std::unique_ptr<ArrowSchema> write_schema, const std::map<std::string, std::string>& options,
+    const std::shared_ptr<MemoryPool>& memory_pool) {
+    return context->GetOrCreateRealtimeStore(
+        RealtimeStoreCreateRequest{std::move(write_schema), options, memory_pool, partition, bucket,
+                                   AppendRealtimeStoreCreateConfig{StatisticsMode::NONE}});
+}
+
+TEST(RealtimeContextTest, TestReusesIndexerAndCapturesRegisteredViews) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
     std::shared_ptr<MemoryPool> pool = GetDefaultPool();
 
-    ASSERT_OK_AND_ASSIGN(
-        RealtimeStoreState first_state,
-        context->GetOrCreateRealtimeStore({{"dt", "2026-08-02"}}, 0, MakeWriteSchema(),
-                                          StatisticsMode::NONE, {{"k", "v"}}, pool));
+    ASSERT_OK_AND_ASSIGN(RealtimeStoreState first_state,
+                         GetOrCreateAppendStore(context, {{"dt", "2026-08-02"}}, 0,
+                                                MakeWriteSchema(), {{"k", "v"}}, pool));
     ASSERT_EQ(0, first_state.initial_offset);
     ASSERT_OK_AND_ASSIGN(
         RealtimeStoreState first_again_state,
-        context->GetOrCreateRealtimeStore({{"dt", "2026-08-02"}}, 0, MakeWriteSchema(),
-                                          StatisticsMode::NONE, {}, pool));
+        GetOrCreateAppendStore(context, {{"dt", "2026-08-02"}}, 0, MakeWriteSchema(), {}, pool));
     ASSERT_EQ(first_state.store, first_again_state.store);
     ASSERT_EQ(0, first_again_state.initial_offset);
     ASSERT_EQ(1, factory->stores.size());
@@ -143,12 +148,10 @@ TEST(RealtimeContextTest, TestReusesStoreAndCapturesRegisteredViews) {
 
     ASSERT_OK_AND_ASSIGN(
         RealtimeStoreState second_state,
-        context->GetOrCreateRealtimeStore({{"dt", "2026-08-02"}}, 1, MakeWriteSchema(),
-                                          StatisticsMode::NONE, {}, pool));
+        GetOrCreateAppendStore(context, {{"dt", "2026-08-02"}}, 1, MakeWriteSchema(), {}, pool));
     ASSERT_OK_AND_ASSIGN(
         RealtimeStoreState third_state,
-        context->GetOrCreateRealtimeStore({{"dt", "2026-08-03"}}, 0, MakeWriteSchema(),
-                                          StatisticsMode::NONE, {}, pool));
+        GetOrCreateAppendStore(context, {{"dt", "2026-08-03"}}, 0, MakeWriteSchema(), {}, pool));
     ASSERT_NE(first_state.store, second_state.store);
     ASSERT_NE(first_state.store, third_state.store);
     ASSERT_EQ(3, factory->stores.size());
@@ -171,10 +174,8 @@ TEST(RealtimeContextTest, TestCommittedProgressIsMonotonicAndSelective) {
     std::shared_ptr<MemoryPool> pool = GetDefaultPool();
     const std::map<std::string, std::string> partition = {{"dt", "2026-08-02"}};
 
-    ASSERT_OK(context->GetOrCreateRealtimeStore(partition, 0, MakeWriteSchema(),
-                                                StatisticsMode::NONE, {}, pool));
-    ASSERT_OK(context->GetOrCreateRealtimeStore(partition, 1, MakeWriteSchema(),
-                                                StatisticsMode::NONE, {}, pool));
+    ASSERT_OK(GetOrCreateAppendStore(context, partition, 0, MakeWriteSchema(), {}, pool));
+    ASSERT_OK(GetOrCreateAppendStore(context, partition, 1, MakeWriteSchema(), {}, pool));
     ASSERT_EQ(2, factory->stores.size());
 
     ASSERT_NOK_WITH_MSG(context->AdvanceCommittedProgress(-1, {}),
@@ -193,8 +194,7 @@ TEST(RealtimeContextTest, TestCommittedProgressIsMonotonicAndSelective) {
 
     ASSERT_OK_AND_ASSIGN(
         RealtimeStoreState restored_state,
-        context->GetOrCreateRealtimeStore({{"dt", "unknown"}}, 0, MakeWriteSchema(),
-                                          StatisticsMode::NONE, {}, pool));
+        GetOrCreateAppendStore(context, {{"dt", "unknown"}}, 0, MakeWriteSchema(), {}, pool));
     ASSERT_EQ(9, restored_state.initial_offset);
 
     ASSERT_OK(context->AdvanceCommittedProgress(
@@ -211,41 +211,15 @@ TEST(RealtimeContextTest, TestCommittedProgressIsMonotonicAndSelective) {
     ASSERT_EQ(std::vector<int64_t>({8}), factory->stores[1]->committed_offsets);
 }
 
-TEST(RealtimeContextTest, TestRemovedInactivePartitionDoesNotRequireReopen) {
-    auto factory = std::make_shared<TestingRealtimeStoreFactory>();
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
-    std::shared_ptr<MemoryPool> pool = GetDefaultPool();
-    const std::map<std::string, std::string> active_partition = {{"dt", "2026-08-02"}};
-    const std::map<std::string, std::string> inactive_partition = {{"dt", "2026-08-03"}};
-    const RealtimePartitionBucket active_partition_bucket(active_partition, /*bucket=*/0);
-    const RealtimePartitionBucket inactive_partition_bucket(inactive_partition, /*bucket=*/0);
-
-    ASSERT_OK(context->AdvanceCommittedProgress(
-        5, {{active_partition_bucket, /*offset=*/7}, {inactive_partition_bucket, /*offset=*/9}}));
-    ASSERT_OK_AND_ASSIGN(RealtimeStoreState active_state,
-                         context->GetOrCreateRealtimeStore(active_partition, 0, MakeWriteSchema(),
-                                                           StatisticsMode::NONE, {}, pool));
-    ASSERT_EQ(7, active_state.initial_offset);
-
-    ASSERT_OK(context->AdvanceCommittedProgress(6, {{active_partition_bucket, /*offset=*/7}}));
-    ASSERT_OK_AND_ASSIGN(RealtimeStoreState inactive_state,
-                         context->GetOrCreateRealtimeStore(inactive_partition, 0, MakeWriteSchema(),
-                                                           StatisticsMode::NONE, {}, pool));
-    ASSERT_EQ(0, inactive_state.initial_offset);
-}
-
 TEST(RealtimeContextTest, TestRetriesOnlyIncompleteReclamation) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
     std::shared_ptr<MemoryPool> pool = GetDefaultPool();
     const std::map<std::string, std::string> partition = {{"dt", "2026-08-02"}};
 
-    ASSERT_OK(context->GetOrCreateRealtimeStore(partition, 0, MakeWriteSchema(),
-                                                StatisticsMode::NONE, {}, pool));
-    ASSERT_OK(context->GetOrCreateRealtimeStore(partition, 1, MakeWriteSchema(),
-                                                StatisticsMode::NONE, {}, pool));
-    ASSERT_OK(context->GetOrCreateRealtimeStore(partition, 2, MakeWriteSchema(),
-                                                StatisticsMode::NONE, {}, pool));
+    ASSERT_OK(GetOrCreateAppendStore(context, partition, 0, MakeWriteSchema(), {}, pool));
+    ASSERT_OK(GetOrCreateAppendStore(context, partition, 1, MakeWriteSchema(), {}, pool));
+    ASSERT_OK(GetOrCreateAppendStore(context, partition, 2, MakeWriteSchema(), {}, pool));
     ASSERT_EQ(3, factory->stores.size());
     factory->stores[1]->fail_next_advance = true;
 
@@ -259,9 +233,9 @@ TEST(RealtimeContextTest, TestRetriesOnlyIncompleteReclamation) {
     ASSERT_TRUE(factory->stores[1]->committed_offsets.empty());
     ASSERT_EQ(std::vector<int64_t>({9}), factory->stores[2]->committed_offsets);
 
-    ASSERT_OK_AND_ASSIGN(RealtimeStoreState failed_store_state,
-                         context->GetOrCreateRealtimeStore(partition, 1, MakeWriteSchema(),
-                                                           StatisticsMode::NONE, {}, pool));
+    ASSERT_OK_AND_ASSIGN(
+        RealtimeStoreState failed_store_state,
+        GetOrCreateAppendStore(context, partition, 1, MakeWriteSchema(), {}, pool));
     ASSERT_EQ(8, failed_store_state.initial_offset);
 
     ASSERT_OK(context->AdvanceCommittedProgress(5, committed_offsets));
@@ -271,45 +245,11 @@ TEST(RealtimeContextTest, TestRetriesOnlyIncompleteReclamation) {
     ASSERT_EQ(std::vector<int64_t>({8}), factory->stores[1]->committed_offsets);
 }
 
-TEST(RealtimeContextTest, TestRequiresReopenWhenCommittedProgressMovesBackwards) {
-    auto factory = std::make_shared<TestingRealtimeStoreFactory>();
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
-    std::shared_ptr<MemoryPool> pool = GetDefaultPool();
-    const std::map<std::string, std::string> first_partition = {{"dt", "2026-08-02"}};
-    const std::map<std::string, std::string> second_partition = {{"dt", "2026-08-03"}};
-    const RealtimePartitionBucket first_partition_bucket(first_partition, /*bucket=*/0);
-    const RealtimePartitionBucket second_partition_bucket(second_partition, /*bucket=*/0);
-
-    ASSERT_OK(context->GetOrCreateRealtimeStore(first_partition, 0, MakeWriteSchema(),
-                                                StatisticsMode::NONE, {}, pool));
-    ASSERT_OK(context->GetOrCreateRealtimeStore(second_partition, 0, MakeWriteSchema(),
-                                                StatisticsMode::NONE, {}, pool));
-    ASSERT_OK(context->AdvanceCommittedProgress(
-        5, {{first_partition_bucket, /*offset=*/7}, {second_partition_bucket, /*offset=*/9}}));
-    ASSERT_EQ(std::vector<int64_t>({7}), factory->stores[0]->committed_offsets);
-    ASSERT_EQ(std::vector<int64_t>({9}), factory->stores[1]->committed_offsets);
-
-    ASSERT_NOK_WITH_MSG(
-        context->AdvanceCommittedProgress(
-            6, {{first_partition_bucket, /*offset=*/6}, {second_partition_bucket, /*offset=*/10}}),
-        "recreate RealtimeContext");
-    ASSERT_NOK_WITH_MSG(
-        context->AdvanceCommittedProgress(6, {{first_partition_bucket, /*offset=*/10}}),
-        "recreate RealtimeContext");
-    ASSERT_EQ(std::vector<int64_t>({7}), factory->stores[0]->committed_offsets);
-    ASSERT_EQ(std::vector<int64_t>({9}), factory->stores[1]->committed_offsets);
-
-    ASSERT_OK(context->AdvanceCommittedProgress(
-        6, {{first_partition_bucket, /*offset=*/10}, {second_partition_bucket, /*offset=*/11}}));
-    ASSERT_EQ(std::vector<int64_t>({7, 10}), factory->stores[0]->committed_offsets);
-    ASSERT_EQ(std::vector<int64_t>({9, 11}), factory->stores[1]->committed_offsets);
-}
-
 TEST(RealtimeContextTest, TestPinsResolvesAndReleasesReadViewTicket) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
-    ASSERT_OK(context->GetOrCreateRealtimeStore(/*partition=*/{}, /*bucket=*/0, MakeWriteSchema(),
-                                                StatisticsMode::NONE, {}, GetDefaultPool()));
+    ASSERT_OK(GetOrCreateAppendStore(context, /*partition=*/{}, /*bucket=*/0, MakeWriteSchema(), {},
+                                     GetDefaultPool()));
     ASSERT_OK_AND_ASSIGN(std::vector<RealtimePartitionBucketView> views,
                          context->AcquireReadViews());
     ASSERT_EQ(1, views.size());
@@ -332,8 +272,8 @@ TEST(RealtimeContextTest, TestPinsResolvesAndReleasesReadViewTicket) {
 TEST(RealtimeContextTest, TestExpiresAbandonedReadViewTicket) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
-    ASSERT_OK(context->GetOrCreateRealtimeStore(/*partition=*/{}, /*bucket=*/0, MakeWriteSchema(),
-                                                StatisticsMode::NONE, {}, GetDefaultPool()));
+    ASSERT_OK(GetOrCreateAppendStore(context, /*partition=*/{}, /*bucket=*/0, MakeWriteSchema(), {},
+                                     GetDefaultPool()));
     ASSERT_OK_AND_ASSIGN(std::vector<RealtimePartitionBucketView> views,
                          context->AcquireReadViews());
     ASSERT_EQ(1, views.size());
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
new file mode 100644
index 000000000..2ebcede82
--- /dev/null
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/realtime/realtime_primary_key_writer.h"
+
+#include <limits>
+#include <optional>
+#include <utility>
+#include <vector>
+
+#include "arrow/api.h"
+#include "arrow/c/bridge.h"
+#include "paimon/common/reader/concat_batch_reader.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/common/types/row_kind.h"
+#include "paimon/common/utils/arrow/arrow_utils.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/scope_guard.h"
+#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/mergetree/merge_tree_writer.h"
+#include "paimon/core/realtime/realtime_context_impl.h"
+#include "paimon/core/utils/commit_increment.h"
+#include "paimon/macros.h"
+#include "paimon/realtime/realtime_context.h"
+
+namespace paimon {
+
+Result<std::shared_ptr<RealtimePrimaryKeyWriter>> RealtimePrimaryKeyWriter::Create(
+    const std::map<std::string, std::string>& partition, int32_t bucket,
+    std::unique_ptr<::ArrowSchema> write_schema,
+    const std::vector<std::string>& trimmed_primary_keys,
+    const std::shared_ptr<RealtimeContext>& realtime_context,
+    const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
+    const std::map<std::string, std::string>& options,
+    const std::shared_ptr<MemoryPool>& memory_pool, int64_t restore_max_sequence_number) {
+    ScopeGuard schema_guard([schema = write_schema.get()]() {
+        if (schema && schema->release) {
+            ArrowSchemaRelease(schema);
+        }
+    });
+    if (!realtime_context) {
+        return Status::Invalid("PK real-time context is null");
+    }
+    if (!merge_tree_writer) {
+        return Status::Invalid("PK real-time merge-tree writer is null");
+    }
+    if (!write_schema || !write_schema->release) {
+        return Status::Invalid("PK real-time write schema is null");
+    }
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> imported_schema,
+                                      arrow::ImportSchema(write_schema.get()));
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*imported_schema, write_schema.get()));
+    RealtimeStoreCreateRequest request{
+        std::move(write_schema),
+        options,
+        memory_pool,
+        partition,
+        bucket,
+        PrimaryKeyRealtimeStoreCreateConfig{trimmed_primary_keys, restore_max_sequence_number}};
+    schema_guard.Release();
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
+                           RealtimeContextImpl::Cast(realtime_context));
+    PAIMON_ASSIGN_OR_RAISE(RealtimeStoreState store_state,
+                           realtime_context_impl->GetOrCreateRealtimeStore(std::move(request)));
+    return std::shared_ptr<RealtimePrimaryKeyWriter>(
+        new RealtimePrimaryKeyWriter(store_state.store, merge_tree_writer, realtime_context_impl,
+                                     RealtimePartitionBucket(partition, bucket), imported_schema,
+                                     store_state.initial_offset, memory_pool));
+}
+
+RealtimePrimaryKeyWriter::RealtimePrimaryKeyWriter(
+    const std::shared_ptr<RealtimeStore>& realtime_store,
+    const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
+    const std::shared_ptr<RealtimeContextImpl>& realtime_context,
+    const RealtimePartitionBucket& partition_bucket,
+    const std::shared_ptr<arrow::Schema>& write_schema, int64_t next_offset,
+    const std::shared_ptr<MemoryPool>& memory_pool)
+    : memory_pool_(memory_pool),
+      realtime_store_(realtime_store),
+      merge_tree_writer_(merge_tree_writer),
+      realtime_context_(realtime_context),
+      partition_bucket_(partition_bucket),
+      write_schema_(write_schema),
+      next_offset_(next_offset) {}
+
+Status RealtimePrimaryKeyWriter::Write(std::unique_ptr<RecordBatch>&& batch) {
+    if (!batch || !batch->GetData()) {
+        return Status::Invalid("PK real-time write batch is null");
+    }
+    const int64_t row_count = batch->GetData()->length;
+    if (row_count == 0) {
+        return Status::OK();
+    }
+    std::lock_guard<std::mutex> lock(realtime_store_mutex_);
+    if (row_count > std::numeric_limits<int64_t>::max() - next_offset_) {
+        return Status::Invalid("real-time offset range exceeds INT64_MAX");
+    }
+    const OffsetRange range(next_offset_, next_offset_ + row_count);
+    PAIMON_RETURN_NOT_OK(realtime_store_->Write(RealtimeWriteBatch{std::move(batch), range}));
+    next_offset_ += row_count;
+    return Status::OK();
+}
+
+Result<CommitIncrement> RealtimePrimaryKeyWriter::PrepareCommit(bool wait_compaction) {
+    std::lock_guard<std::mutex> lock(prepare_mutex_);
+    std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment;
+    {
+        std::lock_guard<std::mutex> realtime_store_lock(realtime_store_mutex_);
+        PAIMON_ASSIGN_OR_RAISE(std::optional<std::shared_ptr<RealtimeSegmentHandle>> sealed_segment,
+                               realtime_store_->SealForCommit());
+        segment = std::move(sealed_segment);
+    }
+    if (segment) {
+        PAIMON_RETURN_NOT_OK(FlushSegment(segment.value()));
+    }
+    PAIMON_ASSIGN_OR_RAISE(CommitIncrement increment,
+                           merge_tree_writer_->PrepareCommit(wait_compaction));
+    if (segment) {
+        const std::vector<std::shared_ptr<DataFileMeta>>& new_files =
+            increment.GetNewFilesIncrement().NewFiles();
+        if (!new_files.empty()) {
+            realtime_context_->AdvanceMaterializedMaxSequenceNumber(
+                partition_bucket_, DataFileMeta::GetMaxSequenceNumber(new_files));
+        }
+        increment.SetRealtimeOffsetRange(segment.value()->GetOffsetRange());
+    }
+    return increment;
+}
+
+Status RealtimePrimaryKeyWriter::FlushSegment(
+    const std::shared_ptr<RealtimeSegmentHandle>& segment) {
+    PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
+                           realtime_store_->CreateCommitReaders(segment));
+    for (const std::unique_ptr<BatchReader>& reader : readers) {
+        if (!reader) {
+            return Status::Invalid("PK real-time store returned a null commit reader");
+        }
+    }
+    ConcatBatchReader reader(std::move(readers), memory_pool_);
+    ScopeGuard reader_guard([&reader]() { reader.Close(); });
+    const OffsetRange offset_range = segment->GetOffsetRange();
+    int64_t emitted_rows = 0;
+    while (true) {
+        PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader.NextBatch());
+        if (BatchReader::IsEofBatch(batch)) {
+            break;
+        }
+        auto& [c_array, c_schema] = batch;
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> imported,
+                                          arrow::ImportArray(c_array.get(), c_schema.get()));
+        if (!imported || imported->type_id() != arrow::Type::STRUCT) {
+            return Status::Invalid("PK real-time store commit reader returned a non-StructArray");
+        }
+        std::shared_ptr<arrow::StructArray> struct_array =
+            checked_pointer_cast<arrow::StructArray>(imported);
+        std::shared_ptr<arrow::Array> value_kind =
+            struct_array->GetFieldByName(SpecialFields::ValueKind().Name());
+        if (!value_kind || value_kind->type_id() != arrow::Type::INT8) {
+            return Status::Invalid(
+                "PK real-time store commit reader must return an INT8 _VALUE_KIND field");
+        }
+        std::shared_ptr<arrow::Int8Array> encoded_row_kinds =
+            checked_pointer_cast<arrow::Int8Array>(value_kind);
+        std::vector<RecordBatch::RowKind> row_kinds;
+        row_kinds.reserve(static_cast<size_t>(encoded_row_kinds->length()));
+        for (int64_t i = 0; i < encoded_row_kinds->length(); ++i) {
+            if (encoded_row_kinds->IsNull(i)) {
+                return Status::Invalid("PK real-time store commit reader returned a null row kind");
+            }
+            PAIMON_ASSIGN_OR_RAISE(const RowKind* row_kind,
+                                   RowKind::FromByteValue(encoded_row_kinds->Value(i)));
+            row_kinds.push_back(static_cast<RecordBatch::RowKind>(row_kind->ToByteValue()));
+        }
+        PAIMON_ASSIGN_OR_RAISE(struct_array, ArrowUtils::RemoveFieldFromStructArray(
+                                                 struct_array, SpecialFields::ValueKind().Name()));
+        if (!struct_array->type()->Equals(arrow::struct_(write_schema_->fields()))) {
+            return Status::Invalid(
+                "PK real-time store commit reader schema does not match table write schema");
+        }
+        const int64_t row_count = struct_array->length();
+        if (row_count > offset_range.Count() - emitted_rows) {
+            return Status::Invalid(
+                "PK real-time store commit readers returned more rows than the sealed offset "
+                "range");
+        }
+        emitted_rows += row_count;
+        if (row_count == 0) {
+            continue;
+        }
+        auto output = std::make_unique<ArrowArray>();
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*struct_array, output.get()));
+        RecordBatchBuilder builder(output.get());
+        builder.SetRowKinds(row_kinds);
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RecordBatch> record_batch, builder.Finish());
+        PAIMON_RETURN_NOT_OK(merge_tree_writer_->Write(std::move(record_batch)));
+    }
+    if (emitted_rows != offset_range.Count()) {
+        return Status::Invalid(
+            "PK real-time store commit readers returned fewer rows than the sealed offset range");
+    }
+    return Status::OK();
+}
+
+Status RealtimePrimaryKeyWriter::Compact(bool) {
+    return Status::Invalid("PK real-time write does not support explicit compaction");
+}
+
+uint64_t RealtimePrimaryKeyWriter::GetMemoryUsage() const {
+    return realtime_store_->GetMemoryUsage();
+}
+
+Status RealtimePrimaryKeyWriter::FlushMemory() {
+    return Status::OK();
+}
+
+Result<bool> RealtimePrimaryKeyWriter::CompactNotCompleted() {
+    return merge_tree_writer_->CompactNotCompleted();
+}
+
+Status RealtimePrimaryKeyWriter::Sync() {
+    return merge_tree_writer_->Sync();
+}
+
+Status RealtimePrimaryKeyWriter::Close() {
+    return merge_tree_writer_->Close();
+}
+
+std::shared_ptr<Metrics> RealtimePrimaryKeyWriter::GetMetrics() const {
+    return merge_tree_writer_->GetMetrics();
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.h b/src/paimon/core/realtime/realtime_primary_key_writer.h
new file mode 100644
index 000000000..fa057e079
--- /dev/null
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.h
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <vector>
+
+#include "paimon/core/utils/batch_writer.h"
+#include "paimon/realtime/realtime_context.h"
+#include "paimon/realtime/realtime_store.h"
+
+struct ArrowSchema;
+
+namespace arrow {
+class Schema;
+}  // namespace arrow
+
+namespace paimon {
+
+class MemoryPool;
+class MergeTreeWriter;
+class RealtimeContext;
+class RealtimeContextImpl;
+
+/// Primary-key real-time writer backed by an in-memory mutation indexer.
+class RealtimePrimaryKeyWriter final : public BatchWriter {
+ public:
+    static Result<std::shared_ptr<RealtimePrimaryKeyWriter>> Create(
+        const std::map<std::string, std::string>& partition, int32_t bucket,
+        std::unique_ptr<::ArrowSchema> write_schema,
+        const std::vector<std::string>& trimmed_primary_keys,
+        const std::shared_ptr<RealtimeContext>& realtime_context,
+        const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
+        const std::map<std::string, std::string>& options,
+        const std::shared_ptr<MemoryPool>& memory_pool, int64_t restore_max_sequence_number);
+
+    Status Write(std::unique_ptr<RecordBatch>&& batch) override;
+    Result<CommitIncrement> PrepareCommit(bool wait_compaction) override;
+    Status Compact(bool full_compaction) override;
+    uint64_t GetMemoryUsage() const override;
+    Status FlushMemory() override;
+    Result<bool> CompactNotCompleted() override;
+    Status Sync() override;
+    Status Close() override;
+    std::shared_ptr<Metrics> GetMetrics() const override;
+
+ private:
+    RealtimePrimaryKeyWriter(const std::shared_ptr<RealtimeStore>& realtime_store,
+                             const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
+                             const std::shared_ptr<RealtimeContextImpl>& realtime_context,
+                             const RealtimePartitionBucket& partition_bucket,
+                             const std::shared_ptr<arrow::Schema>& write_schema,
+                             int64_t next_offset, const std::shared_ptr<MemoryPool>& memory_pool);
+
+    Status FlushSegment(const std::shared_ptr<RealtimeSegmentHandle>& segment);
+
+    std::shared_ptr<MemoryPool> memory_pool_;
+    std::shared_ptr<RealtimeStore> realtime_store_;
+    std::shared_ptr<MergeTreeWriter> merge_tree_writer_;
+    std::shared_ptr<RealtimeContextImpl> realtime_context_;
+    RealtimePartitionBucket partition_bucket_;
+    std::shared_ptr<arrow::Schema> write_schema_;
+    int64_t next_offset_;
+    std::mutex realtime_store_mutex_;
+    std::mutex prepare_mutex_;
+};
+
+}  // namespace paimon

From 82949b30a63f96645d36ded2d1987f2cedc91c4f Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Wed, 19 Aug 2026 22:34:33 +0800
Subject: [PATCH 02/62] feat(read): merge primary-key realtime memory with
 snapshots

Capture partition-bucket read views in realtime splits and merge PK memory readers with snapshot data by key range.

Retain read views for reader lifetime, defer ticket consumption until vector reader construction succeeds, and apply predicates after PK deduplication.
---
 .../core/operation/merge_file_split_read.cpp  | 274 ++++++++++++++++++
 .../core/operation/merge_file_split_read.h    |  18 ++
 .../table/source/key_value_table_read.cpp     | 264 +++++++++++++++++
 .../core/table/source/key_value_table_read.h  |   7 +
 .../core/table/source/realtime_table_scan.cpp |   2 +-
 .../core/table/source/realtime_table_scan.h   |   2 +-
 src/paimon/core/table/source/table_scan.cpp   |   7 +-
 7 files changed, 569 insertions(+), 5 deletions(-)

diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp
index b753ea431..8d8367e39 100644
--- a/src/paimon/core/operation/merge_file_split_read.cpp
+++ b/src/paimon/core/operation/merge_file_split_read.cpp
@@ -30,6 +30,7 @@
 #include "arrow/c/bridge.h"
 #include "arrow/type.h"
 #include "fmt/format.h"
+#include "paimon/common/metrics/metrics_impl.h"
 #include "paimon/common/reader/complete_row_kind_batch_reader.h"
 #include "paimon/common/reader/concat_batch_reader.h"
 #include "paimon/common/table/special_fields.h"
@@ -78,6 +79,273 @@ struct KeyValue;
 template <typename T>
 class MergeFunctionWrapper;
 
+namespace {
+
+/// Concatenates merge readers whose key ranges are ordered and non-overlapping, preserving one
+/// projection pipeline without merging independent disk-only components.
+class ConcatNonOverlappingMergeReaders final : public SortMergeReader {
+ public:
+    explicit ConcatNonOverlappingMergeReaders(
+        std::vector<std::unique_ptr<SortMergeReader>>&& readers)
+        : readers_(std::move(readers)) {}
+
+    Result<std::unique_ptr<SortMergeReader::Iterator>> NextBatch() override {
+        while (current_ < readers_.size()) {
+            PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<SortMergeReader::Iterator> iterator,
+                                   readers_[current_]->NextBatch());
+            if (iterator) {
+                return iterator;
+            }
+            readers_[current_]->Close();
+            ++current_;
+        }
+        return std::unique_ptr<SortMergeReader::Iterator>();
+    }
+
+    void Close() override {
+        while (current_ < readers_.size()) {
+            readers_[current_++]->Close();
+        }
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return MetricsImpl::CollectReadMetrics(readers_);
+    }
+
+ private:
+    std::vector<std::unique_ptr<SortMergeReader>> readers_;
+    size_t current_ = 0;
+};
+
+}  // namespace
+
+class MergeFileSplitRead::RealtimeReaderBuilder {
+ public:
+    static Result<std::unique_ptr<BatchReader>> Create(
+        MergeFileSplitRead* owner, const std::vector<std::shared_ptr<Split>>& disk_splits,
+        std::vector<AdditionalKeyValueReader>&& additional_readers) {
+        RealtimeReaderBuilder builder(owner);
+        if (disk_splits.empty()) {
+            std::vector<std::unique_ptr<KeyValueRecordReader>> readers;
+            readers.reserve(additional_readers.size());
+            for (AdditionalKeyValueReader& additional : additional_readers) {
+                readers.push_back(std::move(additional.reader));
+            }
+            return builder.CreateMergedReader(std::move(readers));
+        }
+
+        PAIMON_RETURN_NOT_OK(builder.CollectDiskInputs(disk_splits));
+        builder.AddRangeInputs(std::move(additional_readers));
+        return builder.CreateReader();
+    }
+
+ private:
+    struct RangeInput {
+        std::shared_ptr<InternalRow> min_key;
+        std::shared_ptr<InternalRow> max_key;
+        std::vector<SortedRun> disk_runs;
+        std::unique_ptr<KeyValueRecordReader> additional_reader;
+    };
+
+    explicit RealtimeReaderBuilder(MergeFileSplitRead* owner) : owner_(owner) {}
+
+    Status CollectDiskInputs(const std::vector<std::shared_ptr<Split>>& disk_splits) {
+        first_split_ = std::dynamic_pointer_cast<DataSplitImpl>(disk_splits.front());
+        if (!first_split_) {
+            return Status::Invalid("merge input disk split is not a data split");
+        }
+        const BinaryRow& partition = first_split_->Partition();
+        const int32_t bucket = first_split_->Bucket();
+        PAIMON_ASSIGN_OR_RAISE(data_file_path_factory_,
+                               owner_->path_factory_->CreateDataFilePathFactory(partition, bucket));
+
+        std::vector<std::shared_ptr<DataFileMeta>> data_files;
+        std::vector<std::optional<DeletionFile>> deletion_files;
+        for (const std::shared_ptr<Split>& disk_split : disk_splits) {
+            std::shared_ptr<DataSplitImpl> data_split =
+                std::dynamic_pointer_cast<DataSplitImpl>(disk_split);
+            if (!data_split || !(data_split->Partition() == partition) ||
+                data_split->Bucket() != bucket) {
+                return Status::Invalid("merge input disk splits do not share a partition-bucket");
+            }
+            if (!data_split->BeforeFiles().empty() || data_split->IsStreaming() ||
+                data_split->Bucket() == BucketModeDefine::POSTPONE_BUCKET) {
+                return Status::Invalid("additional merge input requires fixed-bucket batch splits");
+            }
+            const std::vector<std::shared_ptr<DataFileMeta>>& split_files = data_split->DataFiles();
+            const std::vector<std::optional<DeletionFile>>& split_deletion_files =
+                data_split->DeletionFiles();
+            if (!split_deletion_files.empty() &&
+                split_deletion_files.size() != split_files.size()) {
+                return Status::Invalid(
+                    "merge input disk split deletion files must be empty or match data files");
+            }
+            data_files.insert(data_files.end(), split_files.begin(), split_files.end());
+            if (split_deletion_files.empty()) {
+                deletion_files.insert(deletion_files.end(), split_files.size(), std::nullopt);
+            } else {
+                deletion_files.insert(deletion_files.end(), split_deletion_files.begin(),
+                                      split_deletion_files.end());
+            }
+        }
+
+        dv_factory_ = DeletionVector::CreateFactory(
+            owner_->options_.GetFileSystem(),
+            DeletionVector::CreateDeletionFileMap(data_files, deletion_files), owner_->pool_);
+        std::vector<std::vector<SortedRun>> disk_sections =
+            IntervalPartition(data_files, owner_->key_comparator_).Partition();
+        inputs_.reserve(disk_sections.size());
+        for (std::vector<SortedRun>& section : disk_sections) {
+            std::shared_ptr<DataFileMeta> min_file = section.front().Files().front();
+            std::shared_ptr<DataFileMeta> max_file = min_file;
+            for (const SortedRun& run : section) {
+                for (const std::shared_ptr<DataFileMeta>& file : run.Files()) {
+                    if (owner_->key_comparator_->CompareTo(file->min_key, min_file->min_key) < 0) {
+                        min_file = file;
+                    }
+                    if (owner_->key_comparator_->CompareTo(file->max_key, max_file->max_key) > 0) {
+                        max_file = file;
+                    }
+                }
+            }
+            inputs_.push_back(RangeInput{std::shared_ptr<InternalRow>(min_file, &min_file->min_key),
+                                         std::shared_ptr<InternalRow>(max_file, &max_file->max_key),
+                                         std::move(section), nullptr});
+        }
+        return Status::OK();
+    }
+
+    void AddRangeInputs(std::vector<AdditionalKeyValueReader>&& additional_readers) {
+        inputs_.reserve(inputs_.size() + additional_readers.size());
+        for (AdditionalKeyValueReader& additional : additional_readers) {
+            has_unknown_range_ |= !additional.min_key || !additional.max_key;
+            inputs_.push_back(RangeInput{additional.min_key, additional.max_key,
+                                         /*disk_runs=*/{}, std::move(additional.reader)});
+        }
+    }
+
+    Result<std::unique_ptr<KeyValueRecordReader>> CreateDiskReader(const SortedRun& run) {
+        return owner_->CreateReaderForRun(first_split_->Partition(), run, dv_factory_,
+                                          owner_->predicate_for_keys_, data_file_path_factory_);
+    }
+
+    Result<std::unique_ptr<BatchReader>> CreateMergedReader(
+        std::vector<std::unique_ptr<KeyValueRecordReader>>&& record_readers) {
+        if (record_readers.empty()) {
+            return std::make_unique<ConcatBatchReader>(std::vector<std::unique_ptr<BatchReader>>{},
+                                                       owner_->pool_);
+        }
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<SortMergeReader> sort_merge_reader,
+                               owner_->CreateSortMergeReader(std::move(record_readers)));
+        return CreateProjectedReader(std::move(sort_merge_reader));
+    }
+
+    Result<std::unique_ptr<BatchReader>> CreateProjectedReader(
+        std::unique_ptr<SortMergeReader>&& sort_merge_reader) {
+        if (!owner_->force_keep_delete_) {
+            sort_merge_reader = std::make_unique<DropDeleteReader>(std::move(sort_merge_reader));
+        }
+
+        std::unique_ptr<BatchReader> projection_reader;
+        if (!owner_->context_->EnableMultiThreadRowToBatch()) {
+            PAIMON_ASSIGN_OR_RAISE(
+                projection_reader,
+                KeyValueProjectionReader::Create(
+                    std::move(sort_merge_reader), owner_->raw_read_schema_, owner_->projection_,
+                    owner_->options_.GetReadBatchSize(), owner_->pool_));
+        } else {
+            const int32_t thread_number = owner_->context_->GetRowToBatchThreadNumber();
+            assert(thread_number > 0);
+            projection_reader = std::make_unique<AsyncKeyValueProjectionReader>(
+                std::move(sort_merge_reader), owner_->raw_read_schema_, owner_->projection_,
+                owner_->options_.GetReadBatchSize(), thread_number, owner_->pool_);
+        }
+        PAIMON_ASSIGN_OR_RAISE(projection_reader,
+                               owner_->ApplyPredicateFilterIfNeeded(
+                                   std::move(projection_reader), owner_->context_->GetPredicate()));
+        return std::make_unique<CompleteRowKindBatchReader>(std::move(projection_reader),
+                                                            owner_->pool_);
+    }
+
+    Result<std::unique_ptr<BatchReader>> CreateUnknownRangeReader() {
+        std::vector<std::unique_ptr<KeyValueRecordReader>> readers;
+        for (RangeInput& input : inputs_) {
+            for (const SortedRun& run : input.disk_runs) {
+                PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> disk_reader,
+                                       CreateDiskReader(run));
+                readers.push_back(std::move(disk_reader));
+            }
+            if (input.additional_reader) {
+                readers.push_back(std::move(input.additional_reader));
+            }
+        }
+        return CreateMergedReader(std::move(readers));
+    }
+
+    Result<std::unique_ptr<BatchReader>> CreateKnownRangeReader() {
+        std::sort(inputs_.begin(), inputs_.end(),
+                  [this](const RangeInput& lhs, const RangeInput& rhs) {
+                      return owner_->key_comparator_->CompareTo(*lhs.min_key, *rhs.min_key) < 0;
+                  });
+        std::vector<std::vector<RangeInput>> components;
+        std::shared_ptr<InternalRow> component_max_key;
+        for (RangeInput& input : inputs_) {
+            if (components.empty() ||
+                owner_->key_comparator_->CompareTo(*input.min_key, *component_max_key) > 0) {
+                components.emplace_back();
+                component_max_key = input.max_key;
+            } else if (owner_->key_comparator_->CompareTo(*input.max_key, *component_max_key) > 0) {
+                component_max_key = input.max_key;
+            }
+            components.back().push_back(std::move(input));
+        }
+
+        std::vector<std::unique_ptr<SortMergeReader>> component_readers;
+        component_readers.reserve(components.size());
+        for (std::vector<RangeInput>& component : components) {
+            if (component.size() == 1 && !component.front().additional_reader) {
+                PAIMON_ASSIGN_OR_RAISE(
+                    std::unique_ptr<SortMergeReader> disk_component,
+                    owner_->CreateSortMergeReaderForSection(
+                        component.front().disk_runs, first_split_->Partition(), dv_factory_,
+                        component.front().disk_runs.size() == 1 ? owner_->context_->GetPredicate()
+                                                                : owner_->predicate_for_keys_,
+                        data_file_path_factory_, /*drop_delete=*/false));
+                component_readers.push_back(std::move(disk_component));
+                continue;
+            }
+
+            std::vector<std::unique_ptr<KeyValueRecordReader>> readers;
+            for (RangeInput& input : component) {
+                for (const SortedRun& run : input.disk_runs) {
+                    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> disk_reader,
+                                           CreateDiskReader(run));
+                    readers.push_back(std::move(disk_reader));
+                }
+                if (input.additional_reader) {
+                    readers.push_back(std::move(input.additional_reader));
+                }
+            }
+            PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<SortMergeReader> component_reader,
+                                   owner_->CreateSortMergeReader(std::move(readers)));
+            component_readers.push_back(std::move(component_reader));
+        }
+        return CreateProjectedReader(
+            std::make_unique<ConcatNonOverlappingMergeReaders>(std::move(component_readers)));
+    }
+
+    Result<std::unique_ptr<BatchReader>> CreateReader() {
+        return has_unknown_range_ ? CreateUnknownRangeReader() : CreateKnownRangeReader();
+    }
+
+    MergeFileSplitRead* owner_;
+    std::shared_ptr<DataSplitImpl> first_split_;
+    std::shared_ptr<DataFilePathFactory> data_file_path_factory_;
+    DeletionVector::Factory dv_factory_;
+    std::vector<RangeInput> inputs_;
+    bool has_unknown_range_ = false;
+};
+
 Result<std::unique_ptr<MergeFileSplitRead>> MergeFileSplitRead::Create(
     const std::shared_ptr<FileStorePathFactory>& path_factory,
     const std::shared_ptr<InternalReadContext>& context,
@@ -158,6 +426,12 @@ Result<std::unique_ptr<BatchReader>> MergeFileSplitRead::CreateReader(
     return std::make_unique<CompleteRowKindBatchReader>(std::move(batch_reader), pool_);
 }
 
+Result<std::unique_ptr<BatchReader>> MergeFileSplitRead::CreateRealtimeReader(
+    const std::vector<std::shared_ptr<Split>>& disk_splits,
+    std::vector<AdditionalKeyValueReader>&& additional_readers) {
+    return RealtimeReaderBuilder::Create(this, disk_splits, std::move(additional_readers));
+}
+
 void MergeFileSplitRead::SetMergeFunctionWrapper(
     const std::shared_ptr<MergeFunctionWrapper<KeyValue>>& merge_function_wrapper) {
     merge_function_wrapper_ = merge_function_wrapper;
diff --git a/src/paimon/core/operation/merge_file_split_read.h b/src/paimon/core/operation/merge_file_split_read.h
index 5003cb55a..0824254b7 100644
--- a/src/paimon/core/operation/merge_file_split_read.h
+++ b/src/paimon/core/operation/merge_file_split_read.h
@@ -55,6 +55,7 @@ class FieldsComparator;
 class FileBatchReader;
 class FileStorePathFactory;
 class InternalReadContext;
+class InternalRow;
 class MemoryPool;
 class SchemaManager;
 class SortedRun;
@@ -65,6 +66,12 @@ struct KeyValue;
 template <typename T>
 class MergeFunctionWrapper;
 
+struct AdditionalKeyValueReader {
+    std::unique_ptr<KeyValueRecordReader> reader;
+    std::shared_ptr<InternalRow> min_key;
+    std::shared_ptr<InternalRow> max_key;
+};
+
 /// If the class name below is enclosed in parentheses, it might be present in the read path;
 /// otherwise, it must be present in the read path.
 ///
@@ -116,10 +123,21 @@ class MergeFileSplitRead : public AbstractSplitRead {
         return value_schema_;
     }
 
+    std::shared_ptr<arrow::Schema> GetKeySchema() const {
+        return key_schema_;
+    }
+
+    /// Merges ordinary disk splits with generic additional sorted KeyValue readers.
+    Result<std::unique_ptr<BatchReader>> CreateRealtimeReader(
+        const std::vector<std::shared_ptr<Split>>& disk_splits,
+        std::vector<AdditionalKeyValueReader>&& additional_readers);
+
     void SetMergeFunctionWrapper(
         const std::shared_ptr<MergeFunctionWrapper<KeyValue>>& merge_function_wrapper);
 
  private:
+    class RealtimeReaderBuilder;
+
     Result<std::unique_ptr<BatchReader>> CreateMergeReader(
         const std::shared_ptr<DataSplitImpl>& data_split,
         const std::shared_ptr<DataFilePathFactory>& data_file_path_factory);
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index 208807493..770caf1ca 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -20,12 +20,28 @@
 #include "paimon/core/table/source/key_value_table_read.h"
 
 #include <utility>
+#include <vector>
 
+#include "arrow/api.h"
+#include "arrow/c/bridge.h"
+#include "paimon/common/data/columnar/columnar_row_ref.h"
+#include "paimon/common/reader/concat_batch_reader.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/common/types/row_kind.h"
+#include "paimon/common/utils/arrow/arrow_utils.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/global_index/indexed_split_impl.h"
+#include "paimon/core/key_value.h"
 #include "paimon/core/operation/merge_file_split_read.h"
 #include "paimon/core/operation/raw_file_split_read.h"
+#include "paimon/core/realtime/primary_key_realtime_store.h"
+#include "paimon/core/realtime/realtime_context_impl.h"
+#include "paimon/core/realtime/realtime_reader.h"
 #include "paimon/core/table/source/data_split_impl.h"
 #include "paimon/core/table/source/pk_count_reader.h"
+#include "paimon/core/table/source/realtime_split.h"
 #include "paimon/status.h"
 
 namespace paimon {
@@ -34,6 +50,163 @@ class Executor;
 class FileStorePathFactory;
 class InternalReadContext;
 class MemoryPool;
+struct ColumnarBatchContext;
+
+namespace {
+
+class QueryBatchKeyValueReader final : public KeyValueRecordReader {
+ public:
+    QueryBatchKeyValueReader(std::unique_ptr<BatchReader>&& reader,
+                             const std::shared_ptr<arrow::Schema>& key_schema,
+                             const std::shared_ptr<arrow::Schema>& value_schema,
+                             const std::shared_ptr<MemoryPool>& pool)
+        : reader_(std::move(reader)),
+          key_schema_(key_schema),
+          value_schema_(value_schema),
+          pool_(pool) {}
+
+    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatch() override;
+    std::shared_ptr<Metrics> GetReaderMetrics() const override;
+    void Close() override;
+
+ private:
+    class Iterator;
+
+    std::unique_ptr<BatchReader> reader_;
+    std::shared_ptr<arrow::Schema> key_schema_;
+    std::shared_ptr<arrow::Schema> value_schema_;
+    std::shared_ptr<MemoryPool> pool_;
+    std::shared_ptr<arrow::StructArray> values_;
+    std::shared_ptr<arrow::Int64Array> sequences_;
+    std::shared_ptr<arrow::Int8Array> row_kinds_;
+    std::shared_ptr<ColumnarBatchContext> key_context_;
+    std::shared_ptr<ColumnarBatchContext> value_context_;
+};
+
+class QueryBatchKeyValueReader::Iterator final : public KeyValueRecordReader::Iterator {
+ public:
+    explicit Iterator(QueryBatchKeyValueReader* reader) : reader_(reader) {}
+
+    Result<bool> HasNext() const override {
+        return cursor_ < reader_->values_->length();
+    }
+
+    Result<KeyValue> Next() override {
+        if (reader_->sequences_->IsNull(cursor_) || reader_->row_kinds_->IsNull(cursor_)) {
+            return Status::Invalid("PK merge metadata must not be null");
+        }
+        PAIMON_ASSIGN_OR_RAISE(const RowKind* row_kind,
+                               RowKind::FromByteValue(reader_->row_kinds_->Value(cursor_)));
+        const int64_t sequence = reader_->sequences_->Value(cursor_);
+        std::shared_ptr<InternalRow> key =
+            std::make_shared<ColumnarRowRef>(reader_->key_context_, cursor_);
+        auto value = std::make_unique<ColumnarRowRef>(reader_->value_context_, cursor_++);
+        return KeyValue(row_kind, sequence, KeyValue::UNKNOWN_LEVEL, std::move(key),
+                        std::move(value));
+    }
+
+ private:
+    QueryBatchKeyValueReader* reader_;
+    int64_t cursor_ = 0;
+};
+
+Result<std::unique_ptr<KeyValueRecordReader::Iterator>> QueryBatchKeyValueReader::NextBatch() {
+    PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader_->NextBatch());
+    if (BatchReader::IsEofBatch(batch)) {
+        return std::unique_ptr<KeyValueRecordReader::Iterator>();
+    }
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> imported,
+                                      arrow::ImportArray(batch.first.get(), batch.second.get()));
+    std::shared_ptr<arrow::StructArray> input =
+        std::dynamic_pointer_cast<arrow::StructArray>(imported);
+    if (!input) {
+        return Status::Invalid("PK merge input is not a StructArray");
+    }
+    sequences_ = std::dynamic_pointer_cast<arrow::Int64Array>(
+        input->GetFieldByName(SpecialFields::SequenceNumber().Name()));
+    row_kinds_ = std::dynamic_pointer_cast<arrow::Int8Array>(
+        input->GetFieldByName(SpecialFields::ValueKind().Name()));
+    if (!sequences_ || !row_kinds_) {
+        return Status::Invalid("PK merge input is missing sequence or value-kind metadata");
+    }
+    PAIMON_ASSIGN_OR_RAISE(input, ArrowUtils::RemoveFieldFromStructArray(
+                                      input, SpecialFields::SequenceNumber().Name()));
+    PAIMON_ASSIGN_OR_RAISE(
+        values_, ArrowUtils::RemoveFieldFromStructArray(input, SpecialFields::ValueKind().Name()));
+    if (!ArrowUtils::EqualsIgnoreNullable(values_->type(),
+                                          arrow::struct_(value_schema_->fields()))) {
+        return Status::Invalid("PK merge input value schema does not match the table read schema");
+    }
+    arrow::ArrayVector key_fields;
+    key_fields.reserve(key_schema_->num_fields());
+    for (const std::shared_ptr<arrow::Field>& field : key_schema_->fields()) {
+        std::shared_ptr<arrow::Array> key = values_->GetFieldByName(field->name());
+        if (!key) {
+            return Status::Invalid("PK merge input is missing key field ", field->name());
+        }
+        key_fields.push_back(std::move(key));
+    }
+    key_context_ = std::make_shared<ColumnarBatchContext>(key_fields, pool_);
+    value_context_ = std::make_shared<ColumnarBatchContext>(values_->fields(), pool_);
+    return std::make_unique<Iterator>(this);
+}
+
+std::shared_ptr<Metrics> QueryBatchKeyValueReader::GetReaderMetrics() const {
+    return reader_->GetReaderMetrics();
+}
+
+void QueryBatchKeyValueReader::Close() {
+    values_.reset();
+    sequences_.reset();
+    row_kinds_.reset();
+    key_context_.reset();
+    value_context_.reset();
+    reader_->Close();
+}
+
+Result<std::vector<AdditionalKeyValueReader>> CreateMemoryReaders(
+    const std::shared_ptr<RealtimeSplit>& split, const RealtimePartitionBucketView& memory,
+    const std::shared_ptr<arrow::Schema>& key_schema,
+    const std::shared_ptr<arrow::Schema>& value_schema,
+    const std::shared_ptr<InternalReadContext>& context,
+    const std::shared_ptr<MemoryPool>& memory_pool) {
+    arrow::FieldVector requested_fields = {
+        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())};
+    requested_fields.insert(requested_fields.end(), value_schema->fields().begin(),
+                            value_schema->fields().end());
+    auto c_schema = std::make_unique<ArrowSchema>();
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(
+        arrow::ExportSchema(*arrow::schema(requested_fields), c_schema.get()));
+    ScopeGuard schema_guard([schema = c_schema.get()]() { ArrowSchemaRelease(schema); });
+    RealtimeQueryContext query_context{c_schema.get(), /*predicate=*/nullptr,
+                                       /*enable_predicate_pushdown=*/false};
+    PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> batch_readers,
+                           memory.store->CreateQueryReaders(
+                               memory.read_view, split->CommittedEndOffset(), query_context));
+    if (batch_readers.empty()) {
+        return Status::Invalid("PK real-time store returned no query readers for active memory");
+    }
+    std::vector<AdditionalKeyValueReader> result;
+    result.reserve(batch_readers.size());
+    for (std::unique_ptr<BatchReader>& reader : batch_readers) {
+        if (!reader) {
+            return Status::Invalid("PK real-time store returned a null query reader");
+        }
+        std::shared_ptr<InternalRow> min_key;
+        std::shared_ptr<InternalRow> max_key;
+        if (auto* provider = dynamic_cast<PrimaryKeyRangeProvider*>(reader.get())) {
+            min_key = provider->GetMinKey();
+            max_key = provider->GetMaxKey();
+        }
+        result.push_back(
+            AdditionalKeyValueReader{std::make_unique<QueryBatchKeyValueReader>(
+                                         std::move(reader), key_schema, value_schema, memory_pool),
+                                     std::move(min_key), std::move(max_key)});
+    }
+    return result;
+}
+
+}  // namespace
 
 KeyValueTableRead::KeyValueTableRead(std::vector<std::unique_ptr<SplitRead>>&& split_reads,
                                      const std::shared_ptr<FileStorePathFactory>& path_factory,
@@ -75,6 +248,11 @@ void KeyValueTableRead::ForceKeepDelete(bool force_keep_delete) {
 
 Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateReader(
     const std::shared_ptr<Split>& split) {
+    std::shared_ptr<RealtimeSplit> realtime_split = std::dynamic_pointer_cast<RealtimeSplit>(split);
+    if (realtime_split) {
+        return CreateRealtimeReader(realtime_split, /*release_ticket=*/true);
+    }
+
     std::shared_ptr<Split> dispatch_split = split;
     if (auto indexed_split = std::dynamic_pointer_cast<IndexedSplitImpl>(split)) {
         PAIMON_RETURN_NOT_OK(indexed_split->Validate());
@@ -126,8 +304,94 @@ Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateReader(
     return Status::Invalid("create reader failed, not read match with data split.");
 }
 
+Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateReader(
+    const std::vector<std::shared_ptr<Split>>& splits) {
+    std::vector<std::unique_ptr<BatchReader>> readers;
+    readers.reserve(splits.size());
+    std::vector<std::shared_ptr<RealtimeSplit>> realtime_splits;
+    for (const std::shared_ptr<Split>& split : splits) {
+        std::shared_ptr<RealtimeSplit> realtime_split =
+            std::dynamic_pointer_cast<RealtimeSplit>(split);
+        if (realtime_split) {
+            PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> reader,
+                                   CreateRealtimeReader(realtime_split, /*release_ticket=*/false));
+            readers.push_back(std::move(reader));
+            realtime_splits.push_back(std::move(realtime_split));
+        } else {
+            PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> reader, CreateReader(split));
+            readers.push_back(std::move(reader));
+        }
+    }
+
+    std::unique_ptr<BatchReader> result =
+        std::make_unique<ConcatBatchReader>(std::move(readers), GetMemoryPool());
+    if (!realtime_splits.empty()) {
+        const std::shared_ptr<RealtimeContext> realtime_context = context_->GetRealtimeContext();
+        if (!realtime_context) {
+            return Status::Invalid("reading a real-time split requires a real-time context");
+        }
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
+                               RealtimeContextImpl::Cast(realtime_context));
+        for (const std::shared_ptr<RealtimeSplit>& realtime_split : realtime_splits) {
+            PAIMON_RETURN_NOT_OK(
+                realtime_context_impl->ReleaseReadView(realtime_split->OpaqueTicket()));
+        }
+    }
+    return result;
+}
+
+Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateRealtimeReader(
+    const std::shared_ptr<RealtimeSplit>& realtime_split, bool release_ticket) {
+    if (realtime_split->Version() != RealtimeSplit::kCurrentVersion) {
+        return Status::Invalid("unsupported real-time split version");
+    }
+    const std::shared_ptr<RealtimeContext> realtime_context = context_->GetRealtimeContext();
+    if (!realtime_context) {
+        return Status::Invalid("reading a real-time split requires a real-time context");
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
+                           RealtimeContextImpl::Cast(realtime_context));
+    PAIMON_ASSIGN_OR_RAISE(RealtimePartitionBucketView memory,
+                           realtime_context_impl->ResolveReadView(realtime_split->OpaqueTicket()));
+    const RealtimePartitionBucket expected_partition_bucket(realtime_split->Partition(),
+                                                            realtime_split->Bucket());
+    if (memory.partition_bucket != expected_partition_bucket) {
+        return Status::Invalid("real-time read-view ticket belongs to another partition-bucket");
+    }
+    const std::optional<OffsetRange> memory_range = memory.read_view->GetOffsetRange();
+    if (!memory_range || memory_range->end != realtime_split->MemoryEndOffset()) {
+        return Status::Invalid("real-time read-view ticket does not match the split offset range");
+    }
+    for (const std::unique_ptr<SplitRead>& read : split_reads_) {
+        auto* merge_read = dynamic_cast<MergeFileSplitRead*>(read.get());
+        if (merge_read) {
+            PAIMON_ASSIGN_OR_RAISE(
+                std::vector<AdditionalKeyValueReader> memory_readers,
+                CreateMemoryReaders(realtime_split, memory, merge_read->GetKeySchema(),
+                                    merge_read->GetValueSchema(), context_, GetMemoryPool()));
+            PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> reader,
+                                   merge_read->CreateRealtimeReader(realtime_split->DiskSplits(),
+                                                                    std::move(memory_readers)));
+            PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RealtimeReader> realtime_reader,
+                                   RealtimeReader::Create(memory.read_view, std::move(reader)));
+            if (release_ticket) {
+                PAIMON_RETURN_NOT_OK(
+                    realtime_context_impl->ReleaseReadView(realtime_split->OpaqueTicket()));
+            }
+            return std::unique_ptr<BatchReader>(std::move(realtime_reader));
+        }
+    }
+    return Status::Invalid("create reader failed, merge file split read not found");
+}
+
 Result<std::unique_ptr<CountReader>> KeyValueTableRead::CreateCountReader(
     const std::vector<std::shared_ptr<Split>>& splits) {
+    for (const std::shared_ptr<Split>& split : splits) {
+        if (std::dynamic_pointer_cast<RealtimeSplit>(split)) {
+            return Status::NotImplemented(
+                "CreateCountReader does not support process-local real-time splits");
+        }
+    }
     if (context_->GetPredicate() != nullptr) {
         return Status::NotImplemented(
             "CreateCountReader with predicate pushdown is not supported yet");
diff --git a/src/paimon/core/table/source/key_value_table_read.h b/src/paimon/core/table/source/key_value_table_read.h
index d6a1c83d3..6824ae59e 100644
--- a/src/paimon/core/table/source/key_value_table_read.h
+++ b/src/paimon/core/table/source/key_value_table_read.h
@@ -35,6 +35,7 @@ class Executor;
 class FileStorePathFactory;
 class InternalReadContext;
 class MemoryPool;
+class RealtimeSplit;
 
 class KeyValueTableRead : public TableRead {
  public:
@@ -45,6 +46,9 @@ class KeyValueTableRead : public TableRead {
 
     Result<std::unique_ptr<BatchReader>> CreateReader(const std::shared_ptr<Split>& split) override;
 
+    Result<std::unique_ptr<BatchReader>> CreateReader(
+        const std::vector<std::shared_ptr<Split>>& splits) override;
+
     Result<std::unique_ptr<CountReader>> CreateCountReader(
         const std::vector<std::shared_ptr<Split>>& splits) override;
 
@@ -57,6 +61,9 @@ class KeyValueTableRead : public TableRead {
                       const std::shared_ptr<MemoryPool>& memory_pool,
                       const std::shared_ptr<Executor>& executor);
 
+    Result<std::unique_ptr<BatchReader>> CreateRealtimeReader(
+        const std::shared_ptr<RealtimeSplit>& realtime_split, bool release_ticket);
+
     std::vector<std::unique_ptr<SplitRead>> split_reads_;
     std::shared_ptr<FileStorePathFactory> path_factory_;
     std::shared_ptr<InternalReadContext> context_;
diff --git a/src/paimon/core/table/source/realtime_table_scan.cpp b/src/paimon/core/table/source/realtime_table_scan.cpp
index c275208c5..1b496d8a1 100644
--- a/src/paimon/core/table/source/realtime_table_scan.cpp
+++ b/src/paimon/core/table/source/realtime_table_scan.cpp
@@ -108,7 +108,7 @@ Result<std::vector<std::shared_ptr<Split>>> RealtimeTableScan::CreateRealtimeSpl
     for (const std::shared_ptr<Split>& split : disk_splits) {
         std::shared_ptr<DataSplitImpl> data_split = std::dynamic_pointer_cast<DataSplitImpl>(split);
         if (!data_split) {
-            return Status::Invalid("real-time append scan requires process-local data splits");
+            return Status::Invalid("real-time scan requires process-local data splits");
         }
         std::vector<std::pair<std::string, std::string>> partition_values;
         PAIMON_ASSIGN_OR_RAISE(partition_values,
diff --git a/src/paimon/core/table/source/realtime_table_scan.h b/src/paimon/core/table/source/realtime_table_scan.h
index 7d036d420..959203ca4 100644
--- a/src/paimon/core/table/source/realtime_table_scan.h
+++ b/src/paimon/core/table/source/realtime_table_scan.h
@@ -35,7 +35,7 @@ class FileSystem;
 class ScanFilter;
 class SnapshotManager;
 
-/// Adds process-local memory splits to a normal append-table batch scan.
+/// Adds process-local memory splits to a normal data-table batch scan.
 class RealtimeTableScan : public TableScan {
  public:
     RealtimeTableScan(std::unique_ptr<TableScan>&& disk_scan,
diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp
index 2dda955ac..b12e59a84 100644
--- a/src/paimon/core/table/source/table_scan.cpp
+++ b/src/paimon/core/table/source/table_scan.cpp
@@ -41,6 +41,7 @@
 #include "paimon/core/operation/data_evolution_file_store_scan.h"
 #include "paimon/core/operation/file_store_scan.h"
 #include "paimon/core/operation/key_value_file_store_scan.h"
+#include "paimon/core/realtime/primary_key_realtime_options.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
 #include "paimon/core/schema/schema_manager.h"
 #include "paimon/core/schema/schema_validation.h"
@@ -225,15 +226,15 @@ Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions&
     if (!core_options.RealtimeEnabled()) {
         return Status::Invalid("real-time scan requires realtime.enabled=true");
     }
-    if (!table_schema.PrimaryKeys().empty()) {
-        return Status::Invalid("real-time union read currently supports append tables only");
-    }
     if (core_options.GetBucket() <= 0) {
         return Status::Invalid("real-time union read requires fixed bucket mode");
     }
     if (core_options.DataEvolutionEnabled()) {
         return Status::Invalid("real-time union read does not support data evolution");
     }
+    if (!table_schema.PrimaryKeys().empty()) {
+        PAIMON_RETURN_NOT_OK(ValidatePrimaryKeyRealtimeOptions(core_options));
+    }
     if (context.IsStreamingMode()) {
         return Status::Invalid("real-time union read currently supports batch scans only");
     }

From f8078a5588ca4ab3dcbef00d37607a2191dcecfc Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Wed, 19 Aug 2026 22:34:33 +0800
Subject: [PATCH 03/62] test(realtime): cover primary-key realtime lifecycle

Cover PK write and read, recovery, external compaction, supported concurrency, writer handoff, ticket lifecycle, plugin contracts, rolling files, and multi-partition and bucket restore.
---
 .../operation/key_value_file_store_write.cpp  |   35 +-
 test/inte/realtime_write_inte_test.cpp        | 1005 ++++++++++++++++-
 2 files changed, 977 insertions(+), 63 deletions(-)

diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp
index 4456ee1c2..e94c45a15 100644
--- a/src/paimon/core/operation/key_value_file_store_write.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write.cpp
@@ -90,20 +90,6 @@ KeyValueFileStoreWrite::KeyValueFileStoreWrite(
     }
 }
 
-Status KeyValueFileStoreWrite::RefreshCommittedSnapshot(int64_t snapshot_id) {
-    if (!realtime_context_) {
-        return Status::Invalid("refresh committed snapshot requires a real-time writer");
-    }
-    PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, snapshot_manager_->LoadSnapshot(snapshot_id));
-    PAIMON_ASSIGN_OR_RAISE(
-        RealtimeOffsetMap committed_offsets,
-        RealtimeCommitProperties::ReadOffsets(std::optional<Snapshot>(std::move(snapshot)),
-                                              options_.GetFileSystem()));
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
-                           RealtimeContextImpl::Cast(realtime_context_));
-    return realtime_context_impl->AdvanceCommittedProgress(snapshot_id, committed_offsets);
-}
-
 Result<std::unique_ptr<FileStoreScan>> KeyValueFileStoreWrite::CreateFileStoreScan(
     const std::shared_ptr<ScanFilter>& scan_filter) const {
     PAIMON_ASSIGN_OR_RAISE(
@@ -139,6 +125,7 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
         Levels::Create(key_comparator_, restore_data_files, options_.GetNumLevels()));
     std::map<std::string, std::string> partition_map;
     int64_t materialized_max_sequence_number = restore_max_seq_number;
+    std::shared_ptr<CompactManager> compact_manager;
     if (realtime_context_) {
         std::vector<std::pair<std::string, std::string>> partition_values;
         PAIMON_ASSIGN_OR_RAISE(partition_values,
@@ -147,15 +134,11 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
             std::map<std::string, std::string>(partition_values.begin(), partition_values.end());
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
                                RealtimeContextImpl::Cast(realtime_context_));
-        const RealtimePartitionBucket partition_bucket(partition_map, bucket);
         materialized_max_sequence_number = realtime_context_impl->GetMaterializedMaxSequenceNumber(
-            partition_bucket, restore_max_seq_number);
+            RealtimePartitionBucket(partition_map, bucket), restore_max_seq_number);
         if (materialized_max_sequence_number == std::numeric_limits<int64_t>::max()) {
             return Status::Invalid("PK sequence number has reached INT64_MAX");
         }
-    }
-    std::shared_ptr<CompactManager> compact_manager;
-    if (realtime_context_) {
         compact_manager = std::make_shared<NoopCompactManager>();
     } else {
         auto compact_strategy = compact_manager_factory_->CreateCompactStrategy();
@@ -181,6 +164,20 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
         writer, options_.ToMap(), pool_, materialized_max_sequence_number);
 }
 
+Status KeyValueFileStoreWrite::RefreshCommittedSnapshot(int64_t snapshot_id) {
+    if (!realtime_context_) {
+        return Status::Invalid("refresh committed snapshot requires a real-time writer");
+    }
+    PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, snapshot_manager_->LoadSnapshot(snapshot_id));
+    PAIMON_ASSIGN_OR_RAISE(
+        RealtimeOffsetMap committed_offsets,
+        RealtimeCommitProperties::ReadOffsets(std::optional<Snapshot>(std::move(snapshot)),
+                                              options_.GetFileSystem()));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
+                           RealtimeContextImpl::Cast(realtime_context_));
+    return realtime_context_impl->AdvanceCommittedProgress(snapshot_id, committed_offsets);
+}
+
 Status KeyValueFileStoreWrite::Close() {
     PAIMON_RETURN_NOT_OK(AbstractFileStoreWrite::Close());
     compact_manager_factory_->Close();
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index 6298137ea..f18c3f1e4 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -23,6 +23,7 @@
 #include <condition_variable>
 #include <cstdint>
 #include <deque>
+#include <future>
 #include <iterator>
 #include <limits>
 #include <map>
@@ -42,7 +43,11 @@
 #include "paimon/catalog/catalog.h"
 #include "paimon/catalog/identifier.h"
 #include "paimon/commit_context.h"
+#include "paimon/common/factories/io_hook.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/path_util.h"
+#include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/operation/commit/realtime_commit_properties.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
@@ -59,6 +64,7 @@
 #include "paimon/predicate/predicate_builder.h"
 #include "paimon/read_context.h"
 #include "paimon/reader/count_reader.h"
+#include "paimon/realtime/arrow_realtime_store_factory.h"
 #include "paimon/realtime/realtime_context.h"
 #include "paimon/realtime/realtime_store.h"
 #include "paimon/record_batch.h"
@@ -71,6 +77,308 @@
 #include "paimon/write_context.h"
 
 namespace paimon::test {
+namespace {
+
+class BlockingState {
+ public:
+    void Block() {
+        std::unique_lock<std::mutex> lock(mutex_);
+        entered_ = true;
+        entered_cv_.notify_all();
+        release_cv_.wait(lock, [this]() { return released_; });
+    }
+
+    bool WaitUntilBlocked() {
+        std::unique_lock<std::mutex> lock(mutex_);
+        return entered_cv_.wait_for(lock, std::chrono::seconds(30), [this]() { return entered_; });
+    }
+
+    void Release() {
+        std::lock_guard<std::mutex> lock(mutex_);
+        released_ = true;
+        release_cv_.notify_all();
+    }
+
+ private:
+    std::mutex mutex_;
+    std::condition_variable entered_cv_;
+    std::condition_variable release_cv_;
+    bool entered_ = false;
+    bool released_ = false;
+};
+
+class BlockingBatchReader final : public BatchReader {
+ public:
+    BlockingBatchReader(std::unique_ptr<BatchReader>&& reader,
+                        const std::shared_ptr<BlockingState>& state)
+        : reader_(std::move(reader)), state_(state) {}
+
+    Result<ReadBatch> NextBatch() override {
+        if (!blocked_) {
+            blocked_ = true;
+            state_->Block();
+        }
+        return reader_->NextBatch();
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return reader_->GetReaderMetrics();
+    }
+
+    void Close() override {
+        reader_->Close();
+    }
+
+ private:
+    std::unique_ptr<BatchReader> reader_;
+    std::shared_ptr<BlockingState> state_;
+    bool blocked_ = false;
+};
+
+class BlockingRealtimeStore final : public RealtimeStore {
+ public:
+    BlockingRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
+                          const std::shared_ptr<BlockingState>& state)
+        : delegate_(delegate), state_(state) {}
+
+    Status Write(RealtimeWriteBatch&& batch) override {
+        return delegate_->Write(std::move(batch));
+    }
+
+    Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>> SealForCommit() override {
+        return delegate_->SealForCommit();
+    }
+
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
+        const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
+        PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
+                               delegate_->CreateCommitReaders(segment));
+        if (!readers.empty()) {
+            readers[0] = std::make_unique<BlockingBatchReader>(std::move(readers[0]), state_);
+        }
+        return readers;
+    }
+
+    Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
+        return delegate_->AcquireReadView();
+    }
+
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
+        const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
+        const RealtimeQueryContext& context) override {
+        return delegate_->CreateQueryReaders(view, offset_begin, context);
+    }
+
+    Status AdvanceCommittedOffset(int64_t committed_offset) override {
+        return delegate_->AdvanceCommittedOffset(committed_offset);
+    }
+
+    uint64_t GetMemoryUsage() const override {
+        return delegate_->GetMemoryUsage();
+    }
+
+ private:
+    std::shared_ptr<RealtimeStore> delegate_;
+    std::shared_ptr<BlockingState> state_;
+};
+
+class BlockingRealtimeStoreFactory final : public RealtimeStoreFactory {
+ public:
+    explicit BlockingRealtimeStoreFactory(const std::shared_ptr<BlockingState>& state)
+        : state_(state) {}
+
+    Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> delegate,
+                               delegate_.Create(std::move(request)));
+        return std::shared_ptr<RealtimeStore>(
+            std::make_shared<BlockingRealtimeStore>(delegate, state_));
+    }
+
+ private:
+    ArrowRealtimeStoreFactory delegate_;
+    std::shared_ptr<BlockingState> state_;
+};
+
+class TrackingRealtimeReadView final : public RealtimeReadView {
+ public:
+    explicit TrackingRealtimeReadView(std::shared_ptr<RealtimeReadView> delegate)
+        : delegate_(std::move(delegate)) {}
+
+    std::optional<OffsetRange> GetOffsetRange() const override {
+        return delegate_->GetOffsetRange();
+    }
+
+    const std::shared_ptr<RealtimeReadView>& Delegate() const {
+        return delegate_;
+    }
+
+ private:
+    std::shared_ptr<RealtimeReadView> delegate_;
+};
+
+class ReadViewCheckingBatchReader final : public BatchReader {
+ public:
+    ReadViewCheckingBatchReader(std::unique_ptr<BatchReader> delegate,
+                                std::weak_ptr<RealtimeReadView> read_view)
+        : delegate_(std::move(delegate)), read_view_(std::move(read_view)) {}
+
+    Result<ReadBatch> NextBatch() override {
+        if (read_view_.expired()) {
+            return Status::Invalid("real-time read view was released before reader completion");
+        }
+        return delegate_->NextBatch();
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return delegate_->GetReaderMetrics();
+    }
+
+    void Close() override {
+        delegate_->Close();
+    }
+
+ private:
+    std::unique_ptr<BatchReader> delegate_;
+    std::weak_ptr<RealtimeReadView> read_view_;
+};
+
+class QueryTrackingRealtimeStore final : public RealtimeStore {
+ public:
+    QueryTrackingRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
+                               const std::shared_ptr<std::atomic<bool>>& saw_query_predicate,
+                               const std::shared_ptr<std::weak_ptr<RealtimeReadView>>& query_view)
+        : delegate_(delegate), saw_query_predicate_(saw_query_predicate), query_view_(query_view) {}
+
+    Status Write(RealtimeWriteBatch&& batch) override {
+        return delegate_->Write(std::move(batch));
+    }
+
+    Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>> SealForCommit() override {
+        return delegate_->SealForCommit();
+    }
+
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
+        const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
+        return delegate_->CreateCommitReaders(segment);
+    }
+
+    Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeReadView> delegate_view,
+                               delegate_->AcquireReadView());
+        return std::shared_ptr<RealtimeReadView>(
+            std::make_shared<TrackingRealtimeReadView>(delegate_view));
+    }
+
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
+        const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
+        const RealtimeQueryContext& context) override {
+        if (context.predicate) {
+            saw_query_predicate_->store(true, std::memory_order_release);
+        }
+        *query_view_ = view;
+        std::shared_ptr<TrackingRealtimeReadView> tracking_view =
+            std::dynamic_pointer_cast<TrackingRealtimeReadView>(view);
+        if (!tracking_view) {
+            return Status::Invalid("query tracking store received an unexpected read view");
+        }
+        PAIMON_ASSIGN_OR_RAISE(
+            std::vector<std::unique_ptr<BatchReader>> readers,
+            delegate_->CreateQueryReaders(tracking_view->Delegate(), offset_begin, context));
+        for (std::unique_ptr<BatchReader>& reader : readers) {
+            reader = std::make_unique<ReadViewCheckingBatchReader>(std::move(reader), view);
+        }
+        return readers;
+    }
+
+    Status AdvanceCommittedOffset(int64_t committed_offset) override {
+        return delegate_->AdvanceCommittedOffset(committed_offset);
+    }
+
+    uint64_t GetMemoryUsage() const override {
+        return delegate_->GetMemoryUsage();
+    }
+
+ private:
+    std::shared_ptr<RealtimeStore> delegate_;
+    std::shared_ptr<std::atomic<bool>> saw_query_predicate_;
+    std::shared_ptr<std::weak_ptr<RealtimeReadView>> query_view_;
+};
+
+class QueryTrackingRealtimeStoreFactory final : public RealtimeStoreFactory {
+ public:
+    explicit QueryTrackingRealtimeStoreFactory(
+        const std::shared_ptr<std::atomic<bool>>& saw_query_predicate,
+        const std::shared_ptr<std::weak_ptr<RealtimeReadView>>& query_view)
+        : saw_query_predicate_(saw_query_predicate), query_view_(query_view) {}
+
+    Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> delegate,
+                               delegate_.Create(std::move(request)));
+        return std::shared_ptr<RealtimeStore>(std::make_shared<QueryTrackingRealtimeStore>(
+            delegate, saw_query_predicate_, query_view_));
+    }
+
+ private:
+    ArrowRealtimeStoreFactory delegate_;
+    std::shared_ptr<std::atomic<bool>> saw_query_predicate_;
+    std::shared_ptr<std::weak_ptr<RealtimeReadView>> query_view_;
+};
+
+class InvalidReaderRealtimeStore final : public RealtimeStore {
+ public:
+    explicit InvalidReaderRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate)
+        : delegate_(delegate) {}
+
+    Status Write(RealtimeWriteBatch&& batch) override {
+        return delegate_->Write(std::move(batch));
+    }
+
+    Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>> SealForCommit() override {
+        return delegate_->SealForCommit();
+    }
+
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
+        const std::shared_ptr<RealtimeSegmentHandle>&) override {
+        std::vector<std::unique_ptr<BatchReader>> readers;
+        readers.push_back(nullptr);
+        return readers;
+    }
+
+    Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
+        return delegate_->AcquireReadView();
+    }
+
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
+        const std::shared_ptr<RealtimeReadView>&, int64_t, const RealtimeQueryContext&) override {
+        return std::vector<std::unique_ptr<BatchReader>>();
+    }
+
+    Status AdvanceCommittedOffset(int64_t committed_offset) override {
+        return delegate_->AdvanceCommittedOffset(committed_offset);
+    }
+
+    uint64_t GetMemoryUsage() const override {
+        return delegate_->GetMemoryUsage();
+    }
+
+ private:
+    std::shared_ptr<RealtimeStore> delegate_;
+};
+
+class InvalidReaderRealtimeStoreFactory final : public RealtimeStoreFactory {
+ public:
+    Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> delegate,
+                               delegate_.Create(std::move(request)));
+        return std::shared_ptr<RealtimeStore>(
+            std::make_shared<InvalidReaderRealtimeStore>(delegate));
+    }
+
+ private:
+    ArrowRealtimeStoreFactory delegate_;
+};
+
+}  // namespace
 
 namespace {
 
@@ -219,6 +527,18 @@ class RealtimeWriteInteTest : public ::testing::Test {
                                        /*ignore_if_exists=*/false));
     }
 
+    void CreatePkTable(const std::vector<std::string>& partition_keys = {}) const {
+        auto c_schema = std::make_unique<ArrowSchema>();
+        ASSERT_TRUE(arrow::ExportSchema(*schema_, c_schema.get()).ok());
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<Catalog> catalog,
+                             Catalog::Create(dir_->Str(), options_));
+        ASSERT_OK(catalog->CreateDatabase("foo", {}, /*ignore_if_exists=*/false));
+        std::vector<std::string> primary_keys = partition_keys;
+        primary_keys.push_back("id");
+        ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), c_schema.get(), partition_keys,
+                                       primary_keys, options_, /*ignore_if_exists=*/false));
+    }
+
     Result<std::unique_ptr<FileStoreWrite>> CreateRealtimeWriter(
         const std::shared_ptr<RealtimeContext>& realtime_context) const {
         WriteContextBuilder builder(table_path_, commit_user_);
@@ -240,6 +560,12 @@ class RealtimeWriteInteTest : public ::testing::Test {
 
     Result<std::unique_ptr<RecordBatch>> MakeBatch(const std::vector<Row>& rows, bool partitioned,
                                                    int32_t bucket) const {
+        return MakeBatch(rows, partitioned, bucket, /*row_kinds=*/{});
+    }
+
+    Result<std::unique_ptr<RecordBatch>> MakeBatch(
+        const std::vector<Row>& rows, bool partitioned, int32_t bucket,
+        const std::vector<RecordBatch::RowKind>& row_kinds) const {
         if (rows.empty()) {
             return Status::Invalid("cannot create an empty test batch");
         }
@@ -263,6 +589,7 @@ class RealtimeWriteInteTest : public ::testing::Test {
         ArrowArray c_array;
         PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array));
         RecordBatchBuilder builder(&c_array);
+        builder.SetRowKinds(row_kinds);
         if (partitioned) {
             builder.SetPartition({{"pt", partition}});
         }
@@ -426,6 +753,16 @@ class RealtimeWriteInteTest : public ::testing::Test {
         return commit->Expire();
     }
 
+    Status CommitMessages(const std::vector<std::shared_ptr<CommitMessage>>& messages,
+                          int64_t commit_identifier) const {
+        CommitContextBuilder builder(table_path_, commit_user_);
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<CommitContext> context,
+                               builder.SetOptions(options_).Finish());
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreCommit> commit,
+                               FileStoreCommit::Create(std::move(context)));
+        return commit->Commit(messages, commit_identifier);
+    }
+
     Result<std::shared_ptr<Plan>> CreatePlan(
         const std::shared_ptr<RealtimeContext>& realtime_context,
         const std::shared_ptr<Predicate>& predicate) const {
@@ -573,6 +910,75 @@ class RealtimeWriteInteTest : public ::testing::Test {
         return memory_usage;
     }
 
+    Result<std::vector<int64_t>> ReadPkSequences(
+        const std::shared_ptr<RealtimeContext>& realtime_context) const {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
+                               RealtimeContextImpl::Cast(realtime_context));
+        PAIMON_ASSIGN_OR_RAISE(std::vector<RealtimePartitionBucketView> views,
+                               realtime_context_impl->AcquireReadViews());
+        if (views.size() != 1) {
+            return Status::Invalid("expected one PK real-time read view");
+        }
+        auto read_schema = std::make_unique<ArrowSchema>();
+        arrow::FieldVector requested_fields = {
+            DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())};
+        requested_fields.insert(requested_fields.end(), schema_->fields().begin(),
+                                schema_->fields().end());
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(
+            arrow::ExportSchema(*arrow::schema(requested_fields), read_schema.get()));
+        ScopeGuard schema_guard([schema = read_schema.get()]() { ArrowSchemaRelease(schema); });
+        RealtimeQueryContext query_context{read_schema.get(), /*predicate=*/nullptr,
+                                           /*enable_predicate_pushdown=*/false};
+        PAIMON_ASSIGN_OR_RAISE(
+            std::vector<std::unique_ptr<BatchReader>> readers,
+            views[0].store->CreateQueryReaders(views[0].read_view,
+                                               /*offset_begin=*/0, query_context));
+        std::vector<int64_t> sequences;
+        for (const std::unique_ptr<BatchReader>& reader : readers) {
+            while (true) {
+                PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader->NextBatch());
+                if (BatchReader::IsEofBatch(batch)) {
+                    break;
+                }
+                PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+                    std::shared_ptr<arrow::Array> imported,
+                    arrow::ImportArray(batch.first.get(), batch.second.get()));
+                std::shared_ptr<arrow::StructArray> values =
+                    std::dynamic_pointer_cast<arrow::StructArray>(imported);
+                if (!values) {
+                    return Status::Invalid("PK query reader did not return a StructArray");
+                }
+                std::shared_ptr<arrow::Int64Array> sequence_array =
+                    std::dynamic_pointer_cast<arrow::Int64Array>(
+                        values->GetFieldByName(SpecialFields::SequenceNumber().Name()));
+                if (!sequence_array) {
+                    return Status::Invalid("PK query reader did not return sequence numbers");
+                }
+                for (int64_t row = 0; row < sequence_array->length(); ++row) {
+                    sequences.push_back(sequence_array->Value(row));
+                }
+            }
+            reader->Close();
+        }
+        return sequences;
+    }
+
+    static std::vector<std::shared_ptr<DataFileMeta>> NewFiles(
+        const std::vector<RealtimeCommitProgress>& progresses) {
+        std::vector<std::shared_ptr<DataFileMeta>> files;
+        for (const RealtimeCommitProgress& progress : progresses) {
+            std::shared_ptr<CommitMessageImpl> message =
+                std::dynamic_pointer_cast<CommitMessageImpl>(progress.commit_message);
+            if (!message) {
+                continue;
+            }
+            const std::vector<std::shared_ptr<DataFileMeta>>& new_files =
+                message->GetNewFilesIncrement().NewFiles();
+            files.insert(files.end(), new_files.begin(), new_files.end());
+        }
+        return files;
+    }
+
     static Status ValidateReadPrefix(const std::vector<Row>& rows, int64_t total_rows) {
         std::vector<bool> seen(static_cast<size_t>(total_rows), false);
         int64_t max_id = -1;
@@ -627,7 +1033,6 @@ class RealtimeWriteInteTest : public ::testing::Test {
         options_[Options::PARTITION_GENERATE_LEGACY_NAME] =
             legacy_partition_name_enabled ? "true" : "false";
         CreateTable(/*partition_keys=*/{"pt"});
-
         ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                              RealtimeContext::Create());
         ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
@@ -666,6 +1071,57 @@ class RealtimeWriteInteTest : public ::testing::Test {
         ASSERT_OK(writer->Close());
     }
 
+    void CheckVectorReaderRetry(bool primary_key) {
+        if (primary_key) {
+            CreatePkTable(/*partition_keys=*/{"pt"});
+        } else {
+            CreateTable(/*partition_keys=*/{"pt"});
+        }
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                             RealtimeContext::Create());
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                             CreateRealtimeWriter(realtime_context));
+        std::vector<Row> p0_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p0_batch,
+                             MakeBatch(p0_rows, /*partitioned=*/true));
+        ASSERT_OK(writer->Write(std::move(p0_batch)));
+        std::vector<Row> p1_rows = MakeRows(/*first_id=*/10, /*count=*/3, /*partition=*/"p1");
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p1_batch,
+                             MakeBatch(p1_rows, /*partitioned=*/true));
+        ASSERT_OK(writer->Write(std::move(p1_batch)));
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
+                             CreatePlan(realtime_context, /*predicate=*/nullptr));
+        ASSERT_EQ(2, plan->Splits().size());
+
+        std::vector<std::shared_ptr<Split>> invalid_splits = plan->Splits();
+        std::shared_ptr<RealtimeSplit> second_split =
+            std::dynamic_pointer_cast<RealtimeSplit>(invalid_splits[1]);
+        ASSERT_NE(nullptr, second_split);
+        std::vector<std::shared_ptr<Split>> second_disk_splits = second_split->DiskSplits();
+        invalid_splits[1] = std::make_shared<RealtimeSplit>(
+            RealtimeSplit::kCurrentVersion + 1, second_split->SnapshotId(),
+            second_split->Partition(), second_split->Bucket(), std::move(second_disk_splits),
+            second_split->CommittedEndOffset(), second_split->MemoryEndOffset(),
+            second_split->OpaqueTicket());
+
+        ReadContextBuilder read_builder(table_path_);
+        read_builder.SetOptions(options_)
+            .SetReadFieldNames({"id", "payload", "pt"})
+            .WithRealtimeContext(realtime_context)
+            .WithMemoryPool(pool_);
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableRead> table_read,
+                             TableRead::Create(std::move(read_context)));
+        ASSERT_NOK_WITH_MSG(table_read->CreateReader(invalid_splits),
+                            "unsupported real-time split version");
+
+        std::vector<Row> expected_rows = p0_rows;
+        expected_rows.insert(expected_rows.end(), p1_rows.begin(), p1_rows.end());
+        ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(plan, realtime_context));
+        ASSERT_EQ(expected_rows, actual_rows);
+        ASSERT_OK(writer->Close());
+    }
+
     std::unique_ptr<UniqueTestDirectory> dir_;
     std::string table_path_;
     std::string commit_user_ = "realtime_commit_user";
@@ -723,6 +1179,459 @@ TEST_F(RealtimeWriteInteTest, TestAppendCommitAndRead) {
     FinalizeCommitAndCheck(writer.get(), /*realtime_commits=*/{}, /*prepare_identifier=*/0, rows);
 }
 
+TEST_F(RealtimeWriteInteTest, TestPkRead) {
+    CreatePkTable();
+    auto saw_query_predicate = std::make_shared<std::atomic<bool>>(false);
+    auto query_view = std::make_shared<std::weak_ptr<RealtimeReadView>>();
+    auto factory =
+        std::make_shared<QueryTrackingRealtimeStoreFactory>(saw_query_predicate, query_view);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create(factory));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+
+    std::vector<Row> first_rows = {{1, "old", "p0"}, {2, "two", "p0"}};
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
+                         MakeBatch(first_rows, /*partitioned=*/false, /*bucket=*/0,
+                                   {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::INSERT}));
+    ASSERT_OK(writer->Write(std::move(first_batch)));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> update_batch,
+                         MakeBatch({Row{1, "new", "p0"}}, /*partitioned=*/false, /*bucket=*/0,
+                                   {RecordBatch::RowKind::UPDATE_AFTER}));
+    ASSERT_OK(writer->Write(std::move(update_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> memory_rows, ReadRows(realtime_context));
+    ASSERT_EQ((std::vector<Row>{{1, "new", "p0"}, {2, "two", "p0"}}), memory_rows);
+
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_EQ(1, progress.size());
+    ASSERT_OK(Commit(progress, /*commit_identifier=*/0));
+
+    std::vector<Row> second_rows = {{1, "latest", "p0"}, {2, "gone", "p0"}, {3, "three", "p0"}};
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
+                         MakeBatch(second_rows, /*partitioned=*/false, /*bucket=*/0,
+                                   {RecordBatch::RowKind::UPDATE_AFTER,
+                                    RecordBatch::RowKind::DELETE, RecordBatch::RowKind::INSERT}));
+    ASSERT_OK(writer->Write(std::move(second_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> union_rows, ReadRows(realtime_context));
+    ASSERT_EQ((std::vector<Row>{{1, "latest", "p0"}, {3, "three", "p0"}}), union_rows);
+
+    const std::string expected_payload = "new";
+    std::shared_ptr<Predicate> predicate = PredicateBuilder::Equal(
+        /*field_index=*/1, /*field_name=*/"payload", FieldType::STRING,
+        Literal(FieldType::STRING, expected_payload.data(), expected_payload.size()));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> filtered_plan,
+                         CreatePlan(realtime_context, predicate));
+    ASSERT_OK_AND_ASSIGN(
+        CollectedReadResult filtered_result,
+        ReadPlan(filtered_plan, realtime_context, {"id", "payload", "pt"}, predicate,
+                 /*enable_predicate_filter=*/true));
+    ASSERT_EQ(nullptr, filtered_result.data);
+    ASSERT_FALSE(saw_query_predicate->load(std::memory_order_acquire));
+    filtered_result.reader->Close();
+    filtered_result.reader.reset();
+    ASSERT_OK(writer->Close());
+    writer.reset();
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> lifetime_plan,
+                         CreatePlan(realtime_context, /*predicate=*/nullptr));
+    ReadContextBuilder read_builder(table_path_);
+    read_builder.SetOptions(options_)
+        .SetReadFieldNames({"id", "payload", "pt"})
+        .WithRealtimeContext(realtime_context)
+        .WithMemoryPool(pool_);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableRead> table_read,
+                         TableRead::Create(std::move(read_context)));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> reader,
+                         table_read->CreateReader(lifetime_plan->Splits()));
+    ASSERT_FALSE(query_view->expired());
+
+    std::weak_ptr<RealtimeContext> weak_context = realtime_context;
+    table_read.reset();
+    lifetime_plan.reset();
+    realtime_context.reset();
+    ASSERT_TRUE(weak_context.expired());
+    ASSERT_FALSE(query_view->expired());
+    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch read_batch, reader->NextBatch());
+    ASSERT_FALSE(BatchReader::IsEofBatch(read_batch));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::Array> read_array,
+                         ReadResultCollector::GetArray(std::move(read_batch)));
+    ASSERT_NE(nullptr, read_array);
+    read_array.reset();
+    reader->Close();
+    reader.reset();
+    ASSERT_TRUE(query_view->expired());
+}
+
+TEST_F(RealtimeWriteInteTest, TestPkRecovery) {
+    CreatePkTable();
+
+    WriteContextBuilder seed_builder(table_path_, commit_user_);
+    seed_builder.SetOptions(options_).WithStreamingMode(true);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> seed_context, seed_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> seed_writer,
+                         FileStoreWrite::Create(std::move(seed_context)));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> seed_batch,
+                         MakeBatch({Row{99, "seed", "p0"}}, /*partitioned=*/false));
+    ASSERT_OK(seed_writer->Write(std::move(seed_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> seed_messages,
+                         seed_writer->PrepareCommit(/*wait_compaction=*/false,
+                                                    /*commit_identifier=*/0));
+    CommitContextBuilder seed_commit_builder(table_path_, commit_user_);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> seed_commit_context,
+                         seed_commit_builder.SetOptions(options_).Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> seed_commit,
+                         FileStoreCommit::Create(std::move(seed_commit_context)));
+    ASSERT_OK(seed_commit->Commit(seed_messages));
+    ASSERT_OK(seed_writer->Close());
+    const std::vector<Row> mutations = {
+        {1, "one", "p0"}, {1, "one-new", "p0"}, {2, "deleted", "p0"}, {3, "three", "p0"}};
+    const std::vector<RecordBatch::RowKind> mutation_kinds = {
+        RecordBatch::RowKind::INSERT, RecordBatch::RowKind::UPDATE_AFTER,
+        RecordBatch::RowKind::DELETE, RecordBatch::RowKind::INSERT};
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> failed_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> failed_writer,
+                         CreateRealtimeWriter(failed_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> failed_batch,
+                         MakeBatch(mutations, /*partitioned=*/false, /*bucket=*/0, mutation_kinds));
+    ASSERT_OK(failed_writer->Write(std::move(failed_batch)));
+    IOHook* io_hook = IOHook::GetInstance();
+    ScopeGuard hook_guard([io_hook]() { io_hook->Clear(); });
+    io_hook->Reset(/*pos=*/0, IOHook::Mode::RETURN_ERROR);
+    Result<std::vector<RealtimeCommitProgress>> failed_prepare =
+        failed_writer->PrepareCommitWithProgress(/*commit_identifier=*/1);
+    io_hook->Clear();
+    ASSERT_TRUE(failed_prepare.status().IsIOError()) << failed_prepare.status().ToString();
+    ASSERT_OK(failed_writer->Close());
+    failed_writer.reset();
+    failed_context.reset();
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_failed_prepare, ReadRows());
+    ASSERT_EQ((std::vector<Row>{{99, "seed", "p0"}}), rows_after_failed_prepare);
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> first_context, RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> first_writer,
+                         CreateRealtimeWriter(first_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         MakeBatch(mutations, /*partitioned=*/false, /*bucket=*/0, mutation_kinds));
+    ASSERT_OK(first_writer->Write(std::move(batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<int64_t> memory_sequences, ReadPkSequences(first_context));
+    ASSERT_EQ((std::vector<int64_t>{2, 3, 4}), memory_sequences);
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+                         first_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_EQ(1, progress.size());
+    ASSERT_EQ(OffsetRange(0, 4), progress[0].offset_range);
+    ASSERT_EQ(1, NewFiles(progress).size());
+    ASSERT_EQ(memory_sequences.front(), NewFiles(progress)[0]->min_sequence_number);
+    ASSERT_EQ(memory_sequences.back(), NewFiles(progress)[0]->max_sequence_number);
+    ASSERT_OK(Commit(progress, /*commit_identifier=*/1));
+    ASSERT_OK(first_writer->Close());
+    first_context.reset();
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_replay, ReadRows());
+    ASSERT_EQ((std::vector<Row>{{1, "one-new", "p0"}, {3, "three", "p0"}, {99, "seed", "p0"}}),
+              rows_after_replay);
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> second_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> second_writer,
+                         CreateRealtimeWriter(second_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> restart_batch,
+                         MakeBatch({Row{4, "four", "p0"}}, /*partitioned=*/false));
+    ASSERT_OK(second_writer->Write(std::move(restart_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<int64_t> restart_sequences, ReadPkSequences(second_context));
+    ASSERT_EQ((std::vector<int64_t>{5}), restart_sequences);
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> restart_progress,
+                         second_writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
+    ASSERT_EQ(1, restart_progress.size());
+    ASSERT_EQ(OffsetRange(4, 5), restart_progress[0].offset_range);
+    ASSERT_EQ(5, NewFiles(restart_progress)[0]->min_sequence_number);
+    ASSERT_EQ(5, NewFiles(restart_progress)[0]->max_sequence_number);
+    ASSERT_OK(second_writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestPkCompaction) {
+    options_[Options::NUM_SORTED_RUNS_COMPACTION_TRIGGER] = "1";
+    CreatePkTable();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+
+    int64_t latest_snapshot_id = -1;
+    constexpr int64_t kCommitRoundsBeforeCompaction = 4;
+    std::set<std::string> committed_file_names;
+    for (int64_t round = 0; round < kCommitRoundsBeforeCompaction; ++round) {
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                             MakeBatch({Row{round, "value-" + std::to_string(round), "p0"}},
+                                       /*partitioned=*/false));
+        ASSERT_OK(writer->Write(std::move(batch)));
+        ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+                             writer->PrepareCommitWithProgress(round));
+        ASSERT_EQ(1, progress.size());
+        std::shared_ptr<CommitMessageImpl> message =
+            std::dynamic_pointer_cast<CommitMessageImpl>(progress[0].commit_message);
+        ASSERT_NE(nullptr, message);
+        ASSERT_TRUE(message->GetCompactIncrement().IsEmpty());
+        ASSERT_EQ(1, NewFiles(progress).size());
+        committed_file_names.insert(NewFiles(progress)[0]->file_name);
+        ASSERT_OK_AND_ASSIGN(latest_snapshot_id, Commit(progress, round));
+        ASSERT_OK(writer->RefreshCommittedSnapshot(latest_snapshot_id));
+        ASSERT_OK_AND_ASSIGN(uint64_t memory_usage, GetRealtimeMemoryUsage(realtime_context));
+        ASSERT_EQ(0, memory_usage);
+    }
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> next_batch,
+                         MakeBatch({Row{4, "value-4", "p0"}},
+                                   /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(next_batch)));
+
+    WriteContextBuilder compact_builder(table_path_, commit_user_);
+    compact_builder.SetOptions(options_).WithStreamingMode(true);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> compact_context, compact_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> compact_writer,
+                         FileStoreWrite::Create(std::move(compact_context)));
+    ASSERT_OK(compact_writer->Compact(/*partition=*/{}, /*bucket=*/0,
+                                      /*full_compaction=*/true));
+    ASSERT_OK_AND_ASSIGN(
+        std::vector<std::shared_ptr<CommitMessage>> compact_messages,
+        compact_writer->PrepareCommit(/*wait_compaction=*/true, /*commit_identifier=*/4));
+    ASSERT_EQ(1, compact_messages.size());
+    std::shared_ptr<CommitMessageImpl> compact_message =
+        std::dynamic_pointer_cast<CommitMessageImpl>(compact_messages[0]);
+    ASSERT_NE(nullptr, compact_message);
+    ASSERT_TRUE(compact_message->GetNewFilesIncrement().IsEmpty());
+    ASSERT_EQ(kCommitRoundsBeforeCompaction,
+              compact_message->GetCompactIncrement().CompactBefore().size());
+    std::set<std::string> compacted_file_names;
+    for (const std::shared_ptr<DataFileMeta>& file :
+         compact_message->GetCompactIncrement().CompactBefore()) {
+        compacted_file_names.insert(file->file_name);
+    }
+    ASSERT_EQ(committed_file_names, compacted_file_names);
+    ASSERT_FALSE(compact_message->GetCompactIncrement().CompactAfter().empty());
+    ASSERT_OK(CommitMessages(compact_messages, /*commit_identifier=*/4));
+    ASSERT_OK(compact_writer->Close());
+
+    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(options_));
+    SnapshotManager snapshot_manager(options.GetFileSystem(), table_path_);
+    ASSERT_OK_AND_ASSIGN(std::optional<Snapshot> compact_snapshot,
+                         snapshot_manager.LatestSnapshot());
+    ASSERT_TRUE(compact_snapshot);
+    ASSERT_EQ(Snapshot::CommitKind::Compact(), compact_snapshot->GetCommitKind());
+    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap offsets, ReadCommittedOffsets());
+    ASSERT_EQ(4, offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
+    ASSERT_OK(writer->RefreshCommittedSnapshot(compact_snapshot->Id()));
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> compacted_rows, ReadRows(realtime_context));
+    ASSERT_EQ((std::vector<Row>{{0, "value-0", "p0"},
+                                {1, "value-1", "p0"},
+                                {2, "value-2", "p0"},
+                                {3, "value-3", "p0"},
+                                {4, "value-4", "p0"}}),
+              compacted_rows);
+
+    constexpr int64_t kCommitRoundsAfterCompaction = 2;
+    for (int64_t round = 0; round < kCommitRoundsAfterCompaction; ++round) {
+        if (round > 0) {
+            ASSERT_OK_AND_ASSIGN(
+                std::unique_ptr<RecordBatch> batch,
+                MakeBatch({Row{4 + round, "value-" + std::to_string(4 + round), "p0"}},
+                          /*partitioned=*/false));
+            ASSERT_OK(writer->Write(std::move(batch)));
+        }
+        const int64_t commit_identifier = 5 + round;
+        ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+                             writer->PrepareCommitWithProgress(commit_identifier));
+        ASSERT_EQ(1, progress.size());
+        ASSERT_EQ(OffsetRange(4 + round, 5 + round), progress[0].offset_range);
+        ASSERT_OK_AND_ASSIGN(latest_snapshot_id, Commit(progress, commit_identifier));
+        ASSERT_OK(writer->RefreshCommittedSnapshot(latest_snapshot_id));
+        ASSERT_OK_AND_ASSIGN(uint64_t memory_usage, GetRealtimeMemoryUsage(realtime_context));
+        ASSERT_EQ(0, memory_usage);
+    }
+    ASSERT_OK_AND_ASSIGN(offsets, ReadCommittedOffsets());
+    ASSERT_EQ(6, offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> final_rows, ReadRows(realtime_context));
+    ASSERT_EQ((std::vector<Row>{{0, "value-0", "p0"},
+                                {1, "value-1", "p0"},
+                                {2, "value-2", "p0"},
+                                {3, "value-3", "p0"},
+                                {4, "value-4", "p0"},
+                                {5, "value-5", "p0"}}),
+              final_rows);
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestPkConcurrency) {
+    CreatePkTable();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+
+    constexpr int64_t kRowCount = 20;
+    constexpr int32_t kReaderCount = 2;
+    std::atomic<bool> writer_done{false};
+    std::atomic<bool> control_done{false};
+    std::atomic<int32_t> commit_count{0};
+    ConcurrentTestState state;
+    std::vector<int32_t> read_counts(kReaderCount, 0);
+
+    std::thread write_thread([&]() {
+        state.WaitForStart();
+        for (int64_t id = 0; id < kRowCount && !state.ShouldStop(); ++id) {
+            Result<std::unique_ptr<RecordBatch>> batch =
+                MakeBatch(MakeRows(id, /*count=*/1, /*partition=*/"p0"),
+                          /*partitioned=*/false);
+            if (state.RecordErrorIfNotOk(batch) ||
+                state.RecordErrorIfNotOk(writer->Write(std::move(batch).value()))) {
+                break;
+            }
+            std::this_thread::sleep_for(std::chrono::milliseconds(2));
+        }
+        writer_done.store(true, std::memory_order_release);
+    });
+
+    std::thread control_thread([&]() {
+        state.WaitForStart();
+        int64_t commit_identifier = 0;
+        do {
+            Result<std::vector<RealtimeCommitProgress>> progress =
+                writer->PrepareCommitWithProgress(commit_identifier);
+            if (state.RecordErrorIfNotOk(progress)) {
+                break;
+            }
+            if (!progress.value().empty()) {
+                Result<int64_t> snapshot = Commit(progress.value(), commit_identifier++);
+                if (state.RecordErrorIfNotOk(snapshot) ||
+                    state.RecordErrorIfNotOk(writer->RefreshCommittedSnapshot(snapshot.value()))) {
+                    break;
+                }
+                commit_count.fetch_add(1, std::memory_order_relaxed);
+            }
+            std::this_thread::sleep_for(std::chrono::milliseconds(1));
+        } while (!writer_done.load(std::memory_order_acquire) && !state.ShouldStop());
+        if (!state.ShouldStop()) {
+            Result<std::vector<RealtimeCommitProgress>> progress =
+                writer->PrepareCommitWithProgress(commit_identifier);
+            if (!state.RecordErrorIfNotOk(progress) && !progress.value().empty()) {
+                Result<int64_t> snapshot = Commit(progress.value(), commit_identifier);
+                if (!state.RecordErrorIfNotOk(snapshot) &&
+                    !state.RecordErrorIfNotOk(writer->RefreshCommittedSnapshot(snapshot.value()))) {
+                    commit_count.fetch_add(1, std::memory_order_relaxed);
+                }
+            }
+        }
+        control_done.store(true, std::memory_order_release);
+    });
+
+    std::vector<std::thread> read_threads;
+    read_threads.reserve(kReaderCount);
+    for (int32_t reader_index = 0; reader_index < kReaderCount; ++reader_index) {
+        read_threads.emplace_back([&, reader_index]() {
+            state.WaitForStart();
+            while (!control_done.load(std::memory_order_acquire) && !state.ShouldStop()) {
+                Result<std::vector<Row>> rows = ReadRows(realtime_context);
+                ++read_counts[reader_index];
+                if (state.RecordErrorIfNotOk(rows) ||
+                    state.RecordErrorIfNotOk(ValidateReadPrefix(rows.value(), kRowCount))) {
+                    break;
+                }
+                std::this_thread::sleep_for(std::chrono::milliseconds(1));
+            }
+        });
+    }
+
+    state.StartWhenReady(/*worker_count=*/2 + kReaderCount);
+    write_thread.join();
+    control_thread.join();
+    for (std::thread& read_thread : read_threads) {
+        read_thread.join();
+    }
+
+    ASSERT_TRUE(state.Errors().empty()) << (state.Errors().empty() ? "" : state.Errors().front());
+    ASSERT_GT(commit_count.load(), 0);
+    for (int32_t read_count : read_counts) {
+        ASSERT_GT(read_count, 0);
+    }
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> final_rows, ReadRows(realtime_context));
+    ASSERT_EQ(kRowCount, static_cast<int64_t>(final_rows.size()));
+    ASSERT_OK(ValidateReadPrefix(final_rows, kRowCount));
+    ASSERT_OK_AND_ASSIGN(uint64_t memory_usage, GetRealtimeMemoryUsage(realtime_context));
+    ASSERT_EQ(0, memory_usage);
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestPkWriteDuringPrepare) {
+    CreatePkTable();
+    auto state = std::make_shared<BlockingState>();
+    auto factory = std::make_shared<BlockingRealtimeStoreFactory>(state);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create(factory));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
+                         MakeBatch({Row{1, "one", "p0"}}, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(first_batch)));
+
+    Result<std::vector<RealtimeCommitProgress>> prepare_result =
+        Status::Invalid("prepare did not run");
+    std::thread prepare_thread(
+        [&]() { prepare_result = writer->PrepareCommitWithProgress(/*commit_identifier=*/0); });
+    const bool prepare_blocked = state->WaitUntilBlocked();
+    if (!prepare_blocked) {
+        state->Release();
+        prepare_thread.join();
+        ASSERT_TRUE(prepare_blocked);
+    }
+
+    std::promise<Status> write_promise;
+    std::future<Status> write_future = write_promise.get_future();
+    std::thread write_thread([&]() {
+        Result<std::unique_ptr<RecordBatch>> batch =
+            MakeBatch({Row{2, "two", "p0"}}, /*partitioned=*/false);
+        if (!batch.ok()) {
+            write_promise.set_value(batch.status());
+            return;
+        }
+        write_promise.set_value(writer->Write(std::move(batch).value()));
+    });
+    const bool write_completed =
+        write_future.wait_for(std::chrono::seconds(5)) == std::future_status::ready;
+    state->Release();
+    prepare_thread.join();
+    write_thread.join();
+
+    ASSERT_TRUE(write_completed);
+    ASSERT_OK(write_future.get());
+    ASSERT_OK(prepare_result);
+    ASSERT_EQ(1, prepare_result.value().size());
+    ASSERT_EQ(OffsetRange(0, 1), prepare_result.value()[0].offset_range);
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_progress,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_EQ(1, second_progress.size());
+    ASSERT_EQ(OffsetRange(1, 2), second_progress[0].offset_range);
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestPkPluginContract) {
+    CreatePkTable();
+    auto factory = std::make_shared<InvalidReaderRealtimeStoreFactory>();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create(factory));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         MakeBatch({Row{1, "one", "p0"}}, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(batch)));
+
+    ASSERT_NOK_WITH_MSG(ReadRows(realtime_context),
+                        "PK real-time store returned no query readers for active memory");
+    ASSERT_NOK_WITH_MSG(writer->PrepareCommitWithProgress(/*commit_identifier=*/0),
+                        "PK real-time store returned a null commit reader");
+    ASSERT_OK(writer->Close());
+}
+
 TEST_F(RealtimeWriteInteTest, TestRollingFilesPreserveProgress) {
     options_[Options::TARGET_FILE_ROW_NUM] = "10";
     CreateTable(/*partition_keys=*/{});
@@ -1235,50 +2144,12 @@ TEST_F(RealtimeWriteInteTest, TestFailedReaderCreationPreservesRealtimeSplitTick
     ASSERT_OK(writer->Close());
 }
 
-TEST_F(RealtimeWriteInteTest, TestVectorReaderFailurePreservesEarlierSplitTicket) {
-    CreateTable(/*partition_keys=*/{"pt"});
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                         CreateRealtimeWriter(realtime_context));
-    std::vector<Row> p0_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p0_batch,
-                         MakeBatch(p0_rows, /*partitioned=*/true));
-    ASSERT_OK(writer->Write(std::move(p0_batch)));
-    std::vector<Row> p1_rows = MakeRows(/*first_id=*/10, /*count=*/3, /*partition=*/"p1");
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p1_batch,
-                         MakeBatch(p1_rows, /*partitioned=*/true));
-    ASSERT_OK(writer->Write(std::move(p1_batch)));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
-                         CreatePlan(realtime_context, /*predicate=*/nullptr));
-    ASSERT_EQ(2, plan->Splits().size());
-
-    std::vector<std::shared_ptr<Split>> invalid_splits = plan->Splits();
-    std::shared_ptr<RealtimeSplit> second_split =
-        std::dynamic_pointer_cast<RealtimeSplit>(invalid_splits[1]);
-    ASSERT_NE(nullptr, second_split);
-    std::vector<std::shared_ptr<Split>> second_disk_splits = second_split->DiskSplits();
-    invalid_splits[1] = std::make_shared<RealtimeSplit>(
-        RealtimeSplit::kCurrentVersion + 1, second_split->SnapshotId(), second_split->Partition(),
-        second_split->Bucket(), std::move(second_disk_splits), second_split->CommittedEndOffset(),
-        second_split->MemoryEndOffset(), second_split->OpaqueTicket());
-
-    ReadContextBuilder read_builder(table_path_);
-    read_builder.SetOptions(options_)
-        .SetReadFieldNames({"id", "payload", "pt"})
-        .WithRealtimeContext(realtime_context)
-        .WithMemoryPool(pool_);
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableRead> table_read,
-                         TableRead::Create(std::move(read_context)));
-    ASSERT_NOK_WITH_MSG(table_read->CreateReader(invalid_splits),
-                        "unsupported real-time split version");
+TEST_F(RealtimeWriteInteTest, TestAppendVectorRetry) {
+    CheckVectorReaderRetry(/*primary_key=*/false);
+}
 
-    std::vector<Row> expected_rows = p0_rows;
-    expected_rows.insert(expected_rows.end(), p1_rows.begin(), p1_rows.end());
-    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(plan, realtime_context));
-    ASSERT_EQ(expected_rows, actual_rows);
-    ASSERT_OK(writer->Close());
+TEST_F(RealtimeWriteInteTest, TestPkVectorRetry) {
+    CheckVectorReaderRetry(/*primary_key=*/true);
 }
 
 TEST_F(RealtimeWriteInteTest, TestCloseWriterKeepsContextReadable) {
@@ -1351,6 +2222,52 @@ TEST_F(RealtimeWriteInteTest, TestCloseWriterAllowsContextReuseByLaterWriter) {
     ASSERT_OK(second_writer->Close());
 }
 
+TEST_F(RealtimeWriteInteTest, TestPkWriterHandoff) {
+    CreatePkTable();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> first_writer,
+                         CreateRealtimeWriter(realtime_context));
+    std::vector<Row> first_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
+                         MakeBatch(first_rows, /*partitioned=*/false));
+    ASSERT_OK(first_writer->Write(std::move(first_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> commits,
+                         first_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_EQ(1, commits.size());
+    ASSERT_EQ(OffsetRange(0, 3), commits[0].offset_range);
+    ASSERT_EQ(1, NewFiles(commits).size());
+    ASSERT_EQ(0, NewFiles(commits)[0]->min_sequence_number);
+    ASSERT_EQ(2, NewFiles(commits)[0]->max_sequence_number);
+    ASSERT_OK(first_writer->Close());
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> second_writer,
+                         CreateRealtimeWriter(realtime_context));
+    std::vector<Row> second_rows = {
+        Row{0, "updated-0", "p0"},
+        Row{3, "value-3", "p0"},
+    };
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
+                         MakeBatch(second_rows, /*partitioned=*/false));
+    ASSERT_OK(second_writer->Write(std::move(second_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_commits,
+                         second_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_EQ(1, second_commits.size());
+    ASSERT_EQ(OffsetRange(3, 5), second_commits[0].offset_range);
+    ASSERT_EQ(1, NewFiles(second_commits).size());
+    ASSERT_EQ(3, NewFiles(second_commits)[0]->min_sequence_number);
+    ASSERT_EQ(4, NewFiles(second_commits)[0]->max_sequence_number);
+
+    commits.push_back(std::move(second_commits[0]));
+    ASSERT_OK(Commit(commits, /*commit_identifier=*/1));
+    std::vector<Row> expected_rows = first_rows;
+    expected_rows[0] = second_rows[0];
+    expected_rows.push_back(second_rows[1]);
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(realtime_context));
+    ASSERT_EQ(expected_rows, actual_rows);
+    ASSERT_OK(second_writer->Close());
+}
+
 TEST_F(RealtimeWriteInteTest, TestReadCommittedDiskAndBuildingMemory) {
     CreateTable(/*partition_keys=*/{});
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,

From 949356aa24dc83150f612067180dbdad749bd037 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Thu, 20 Aug 2026 14:01:40 +0800
Subject: [PATCH 04/62] refactor(realtime): consolidate PK state and validation

---
 src/paimon/CMakeLists.txt                     |  2 -
 .../core/operation/file_store_write.cpp       |  3 +-
 .../operation/key_value_file_store_write.cpp  | 30 ++++++----
 .../realtime/primary_key_realtime_options.cpp | 58 -------------------
 .../realtime/primary_key_realtime_options.h   | 31 ----------
 .../primary_key_realtime_options_test.cpp     | 56 ------------------
 .../core/realtime/realtime_context_impl.cpp   | 27 ++++-----
 .../core/realtime/realtime_context_impl.h     |  4 +-
 .../core/realtime/realtime_context_test.cpp   | 38 ++++++++++++
 .../realtime/realtime_primary_key_writer.cpp  | 41 ++-----------
 .../realtime/realtime_primary_key_writer.h    | 13 ++---
 src/paimon/core/table/source/table_scan.cpp   |  4 +-
 .../core/utils/primary_key_table_utils.cpp    | 32 ++++++++++
 .../core/utils/primary_key_table_utils.h      |  3 +
 .../utils/primary_key_table_utils_test.cpp    | 26 +++++++++
 15 files changed, 144 insertions(+), 224 deletions(-)
 delete mode 100644 src/paimon/core/realtime/primary_key_realtime_options.cpp
 delete mode 100644 src/paimon/core/realtime/primary_key_realtime_options.h
 delete mode 100644 src/paimon/core/realtime/primary_key_realtime_options_test.cpp

diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index ea6159821..d2c0a2b4f 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -383,7 +383,6 @@ set(PAIMON_CORE_SRCS
     core/realtime/arrow_realtime_store.cpp
     core/realtime/arrow_realtime_store_factory.cpp
     core/realtime/primary_key_realtime_store.cpp
-    core/realtime/primary_key_realtime_options.cpp
     core/realtime/realtime_append_only_writer.cpp
     core/realtime/realtime_context.cpp
     core/realtime/realtime_context_impl.cpp
@@ -791,7 +790,6 @@ if(PAIMON_BUILD_TESTS)
                     core/memory/writer_memory_manager_test.cpp
                     core/realtime/arrow_realtime_store_test.cpp
                     core/realtime/primary_key_realtime_store_test.cpp
-                    core/realtime/primary_key_realtime_options_test.cpp
                     core/realtime/realtime_context_test.cpp
                     core/realtime/realtime_reader_test.cpp
                     core/mergetree/levels_test.cpp
diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp
index fb83c254c..f216476bd 100644
--- a/src/paimon/core/operation/file_store_write.cpp
+++ b/src/paimon/core/operation/file_store_write.cpp
@@ -36,7 +36,6 @@
 #include "paimon/core/operation/key_value_file_store_write.h"
 #include "paimon/core/options/merge_engine.h"
 #include "paimon/core/postpone/postpone_bucket_file_store_write.h"
-#include "paimon/core/realtime/primary_key_realtime_options.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
 #include "paimon/core/schema/schema_manager.h"
 #include "paimon/core/schema/table_schema.h"
@@ -198,7 +197,7 @@ Result<std::unique_ptr<FileStoreWrite>> FileStoreWrite::Create(std::unique_ptr<W
     } else {
         // pk table
         if (ctx->GetRealtimeContext()) {
-            PAIMON_RETURN_NOT_OK(ValidatePrimaryKeyRealtimeOptions(options));
+            PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options));
             if (ignore_previous_files) {
                 return Status::NotImplemented(
                     "PK realtime v1 requires restore from the latest snapshot");
diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp
index e94c45a15..492161cf8 100644
--- a/src/paimon/core/operation/key_value_file_store_write.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write.cpp
@@ -124,19 +124,28 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
         std::shared_ptr<Levels> levels,
         Levels::Create(key_comparator_, restore_data_files, options_.GetNumLevels()));
     std::map<std::string, std::string> partition_map;
-    int64_t materialized_max_sequence_number = restore_max_seq_number;
+    int64_t initial_max_sequence_number = restore_max_seq_number;
     std::shared_ptr<CompactManager> compact_manager;
+    std::shared_ptr<RealtimeContextImpl> realtime_context_impl;
+    std::optional<RealtimeStoreState> realtime_store_state;
     if (realtime_context_) {
         std::vector<std::pair<std::string, std::string>> partition_values;
         PAIMON_ASSIGN_OR_RAISE(partition_values,
                                file_store_path_factory_->GeneratePartitionVector(partition));
         partition_map =
             std::map<std::string, std::string>(partition_values.begin(), partition_values.end());
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
-                               RealtimeContextImpl::Cast(realtime_context_));
-        materialized_max_sequence_number = realtime_context_impl->GetMaterializedMaxSequenceNumber(
-            RealtimePartitionBucket(partition_map, bucket), restore_max_seq_number);
-        if (materialized_max_sequence_number == std::numeric_limits<int64_t>::max()) {
+        PAIMON_ASSIGN_OR_RAISE(realtime_context_impl, RealtimeContextImpl::Cast(realtime_context_));
+        auto c_write_schema = std::make_unique<ArrowSchema>();
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema_, c_write_schema.get()));
+        PAIMON_ASSIGN_OR_RAISE(
+            RealtimeStoreState store_state,
+            realtime_context_impl->GetOrCreateRealtimeStore(RealtimeStoreCreateRequest{
+                std::move(c_write_schema), options_.ToMap(), pool_, partition_map, bucket,
+                PrimaryKeyRealtimeStoreCreateConfig{trimmed_primary_keys,
+                                                    restore_max_seq_number}}));
+        realtime_store_state = std::move(store_state);
+        initial_max_sequence_number = realtime_store_state->initial_max_sequence_number.value();
+        if (initial_max_sequence_number == std::numeric_limits<int64_t>::max()) {
             return Status::Invalid("PK sequence number has reached INT64_MAX");
         }
         compact_manager = std::make_shared<NoopCompactManager>();
@@ -150,18 +159,15 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
     PAIMON_ASSIGN_OR_RAISE(
         std::shared_ptr<MergeTreeWriter> writer,
         MergeTreeWriter::Create(
-            materialized_max_sequence_number, trimmed_primary_keys, data_file_path_factory,
+            initial_max_sequence_number, trimmed_primary_keys, data_file_path_factory,
             key_comparator_, user_defined_seq_comparator_, merge_function_wrapper_,
             table_schema_->Id(), schema_, options_, compact_manager,
             realtime_context_ ? nullptr : io_manager_, enable_multi_thread_spill_, pool_));
     if (!realtime_context_) {
         return std::shared_ptr<BatchWriter>(std::move(writer));
     }
-    auto c_write_schema = std::make_unique<ArrowSchema>();
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema_, c_write_schema.get()));
-    return RealtimePrimaryKeyWriter::Create(
-        partition_map, bucket, std::move(c_write_schema), trimmed_primary_keys, realtime_context_,
-        writer, options_.ToMap(), pool_, materialized_max_sequence_number);
+    return RealtimePrimaryKeyWriter::Create(partition_map, bucket, schema_, realtime_context_impl,
+                                            writer, pool_, realtime_store_state.value());
 }
 
 Status KeyValueFileStoreWrite::RefreshCommittedSnapshot(int64_t snapshot_id) {
diff --git a/src/paimon/core/realtime/primary_key_realtime_options.cpp b/src/paimon/core/realtime/primary_key_realtime_options.cpp
deleted file mode 100644
index e9779a59e..000000000
--- a/src/paimon/core/realtime/primary_key_realtime_options.cpp
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include "paimon/core/realtime/primary_key_realtime_options.h"
-
-#include "paimon/core/core_options.h"
-
-namespace paimon {
-
-Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options) {
-    if (options.GetBucket() <= 0) {
-        return Status::NotImplemented("PK realtime v1 requires fixed buckets");
-    }
-    if (options.GetMergeEngine() != MergeEngine::DEDUPLICATE) {
-        return Status::NotImplemented("PK realtime v1 supports only the DEDUPLICATE merge engine");
-    }
-    if (options.DataEvolutionEnabled()) {
-        return Status::NotImplemented("PK realtime v1 does not support data evolution");
-    }
-    if (!options.GetFieldsSequenceGroups().empty()) {
-        return Status::NotImplemented("PK realtime v1 does not support sequence groups");
-    }
-    if (options.IgnoreDelete() || options.PartialUpdateRemoveRecordOnDelete() ||
-        options.AggregationRemoveRecordOnDelete() ||
-        !options.GetPartialUpdateRemoveRecordOnSequenceGroup().empty()) {
-        return Status::NotImplemented("PK realtime v1 requires default delete behavior");
-    }
-    if (!options.GetSequenceField().empty()) {
-        return Status::NotImplemented("PK realtime v1 does not support sequence.field");
-    }
-    if (!options.SequenceFieldSortOrderIsAscending()) {
-        return Status::NotImplemented(
-            "PK realtime v1 supports only ascending sequence.field.sort-order");
-    }
-    if (options.NeedLookup() || options.DeletionVectorsEnabled() ||
-        options.GetChangelogProducer() != ChangelogProducer::NONE) {
-        return Status::NotImplemented("PK realtime v1 does not support lookup or early MOR");
-    }
-    return Status::OK();
-}
-
-}  // namespace paimon
diff --git a/src/paimon/core/realtime/primary_key_realtime_options.h b/src/paimon/core/realtime/primary_key_realtime_options.h
deleted file mode 100644
index a16d35778..000000000
--- a/src/paimon/core/realtime/primary_key_realtime_options.h
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include "paimon/status.h"
-
-namespace paimon {
-
-class CoreOptions;
-
-/// Validates the table options supported by the in-memory PK realtime V1 path.
-Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options);
-
-}  // namespace paimon
diff --git a/src/paimon/core/realtime/primary_key_realtime_options_test.cpp b/src/paimon/core/realtime/primary_key_realtime_options_test.cpp
deleted file mode 100644
index 5d3ea7f67..000000000
--- a/src/paimon/core/realtime/primary_key_realtime_options_test.cpp
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include "paimon/core/realtime/primary_key_realtime_options.h"
-
-#include <map>
-#include <string>
-#include <vector>
-
-#include "paimon/core/core_options.h"
-#include "paimon/defs.h"
-#include "paimon/testing/utils/testharness.h"
-
-namespace paimon::test {
-
-TEST(PrimaryKeyRealtimeOptionsTest, TestSupportedOptions) {
-    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
-    ASSERT_OK(ValidatePrimaryKeyRealtimeOptions(options));
-}
-
-TEST(PrimaryKeyRealtimeOptionsTest, TestUnsupportedOptions) {
-    const std::string sequence_group =
-        std::string(Options::FIELDS_PREFIX) + ".value." + Options::SEQUENCE_GROUP;
-    const std::vector<std::map<std::string, std::string>> unsupported_options = {
-        {{Options::BUCKET, "0"}},
-        {{Options::BUCKET, "1"}, {Options::MERGE_ENGINE, "partial-update"}},
-        {{Options::BUCKET, "1"}, {Options::DATA_EVOLUTION_ENABLED, "true"}},
-        {{Options::BUCKET, "1"}, {sequence_group, "seq"}},
-        {{Options::BUCKET, "1"}, {Options::SEQUENCE_FIELD, "seq"}},
-        {{Options::BUCKET, "1"}, {Options::FORCE_LOOKUP, "true"}},
-        {{Options::BUCKET, "1"}, {Options::DELETION_VECTORS_ENABLED, "true"}},
-        {{Options::BUCKET, "1"}, {Options::CHANGELOG_PRODUCER, "input"}},
-    };
-    for (const std::map<std::string, std::string>& option_map : unsupported_options) {
-        ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
-        ASSERT_NOK(ValidatePrimaryKeyRealtimeOptions(options));
-    }
-}
-
-}  // namespace paimon::test
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index 0a367b2cd..6624059a6 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -82,6 +82,18 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     std::lock_guard<std::mutex> progress_lock(progress_mutex_);
     std::lock_guard<std::mutex> registry_lock(mutex_);
     const RealtimePartitionBucket key(request.partition, request.bucket);
+    std::optional<int64_t> initial_max_sequence_number;
+    PrimaryKeyRealtimeStoreCreateConfig* primary_key_config =
+        std::get_if<PrimaryKeyRealtimeStoreCreateConfig>(&request.mode_config);
+    if (primary_key_config) {
+        auto [sequence_iter, inserted] = materialized_max_sequence_numbers_.emplace(
+            key, primary_key_config->restore_max_sequence_number);
+        if (!inserted && primary_key_config->restore_max_sequence_number > sequence_iter->second) {
+            sequence_iter->second = primary_key_config->restore_max_sequence_number;
+        }
+        initial_max_sequence_number = sequence_iter->second;
+        primary_key_config->restore_max_sequence_number = sequence_iter->second;
+    }
     int64_t initial_offset = 0;
     auto offset_iter = committed_offsets_.find(key);
     if (offset_iter != committed_offsets_.end()) {
@@ -114,7 +126,7 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
                 initial_offset = memory_range->end;
             }
         }
-        return RealtimeStoreState{iter->second, initial_offset};
+        return RealtimeStoreState{iter->second, initial_offset, initial_max_sequence_number};
     }
     Result<std::shared_ptr<RealtimeStore>> store_result = factory_->Create(std::move(request));
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> store, std::move(store_result));
@@ -122,18 +134,7 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     if (offset_iter != committed_offsets_.end()) {
         reclaimed_offsets_.emplace(key, offset_iter->second);
     }
-    return RealtimeStoreState{std::move(store), initial_offset};
-}
-
-int64_t RealtimeContextImpl::GetMaterializedMaxSequenceNumber(
-    const RealtimePartitionBucket& partition_bucket, int64_t restored_max_sequence_number) {
-    std::lock_guard<std::mutex> lock(mutex_);
-    auto [iter, inserted] =
-        materialized_max_sequence_numbers_.emplace(partition_bucket, restored_max_sequence_number);
-    if (!inserted && restored_max_sequence_number > iter->second) {
-        iter->second = restored_max_sequence_number;
-    }
-    return iter->second;
+    return RealtimeStoreState{std::move(store), initial_offset, initial_max_sequence_number};
 }
 
 void RealtimeContextImpl::AdvanceMaterializedMaxSequenceNumber(
diff --git a/src/paimon/core/realtime/realtime_context_impl.h b/src/paimon/core/realtime/realtime_context_impl.h
index 45d07deeb..f4cd3866e 100644
--- a/src/paimon/core/realtime/realtime_context_impl.h
+++ b/src/paimon/core/realtime/realtime_context_impl.h
@@ -47,6 +47,7 @@ class MemoryPool;
 struct RealtimeStoreState {
     std::shared_ptr<RealtimeStore> store;
     int64_t initial_offset;
+    std::optional<int64_t> initial_max_sequence_number;
 };
 
 struct RealtimePartitionBucketView {
@@ -67,9 +68,6 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
 
     Result<RealtimeStoreState> GetOrCreateRealtimeStore(RealtimeStoreCreateRequest&& request);
 
-    int64_t GetMaterializedMaxSequenceNumber(const RealtimePartitionBucket& partition_bucket,
-                                             int64_t restored_max_sequence_number);
-
     void AdvanceMaterializedMaxSequenceNumber(const RealtimePartitionBucket& partition_bucket,
                                               int64_t max_sequence_number);
 
diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp
index 33701afac..b4d2c6718 100644
--- a/src/paimon/core/realtime/realtime_context_test.cpp
+++ b/src/paimon/core/realtime/realtime_context_test.cpp
@@ -129,6 +129,15 @@ Result<RealtimeStoreState> GetOrCreateAppendStore(
                                    AppendRealtimeStoreCreateConfig{StatisticsMode::NONE}});
 }
 
+Result<RealtimeStoreState> GetOrCreatePrimaryKeyStore(
+    const std::shared_ptr<RealtimeContextImpl>& context,
+    const std::map<std::string, std::string>& partition, int32_t bucket,
+    int64_t restore_max_sequence_number, const std::shared_ptr<MemoryPool>& memory_pool) {
+    return context->GetOrCreateRealtimeStore(RealtimeStoreCreateRequest{
+        MakeWriteSchema(), /*options=*/{}, memory_pool, partition, bucket,
+        PrimaryKeyRealtimeStoreCreateConfig{{"id"}, restore_max_sequence_number}});
+}
+
 TEST(RealtimeContextTest, TestReusesIndexerAndCapturesRegisteredViews) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
@@ -138,6 +147,7 @@ TEST(RealtimeContextTest, TestReusesIndexerAndCapturesRegisteredViews) {
                          GetOrCreateAppendStore(context, {{"dt", "2026-08-02"}}, 0,
                                                 MakeWriteSchema(), {{"k", "v"}}, pool));
     ASSERT_EQ(0, first_state.initial_offset);
+    ASSERT_FALSE(first_state.initial_max_sequence_number);
     ASSERT_OK_AND_ASSIGN(
         RealtimeStoreState first_again_state,
         GetOrCreateAppendStore(context, {{"dt", "2026-08-02"}}, 0, MakeWriteSchema(), {}, pool));
@@ -168,6 +178,34 @@ TEST(RealtimeContextTest, TestReusesIndexerAndCapturesRegisteredViews) {
     ASSERT_EQ(1, factory->stores[2]->acquire_count);
 }
 
+TEST(RealtimeContextTest, TestReconcilesPrimaryKeyInitialSequence) {
+    auto factory = std::make_shared<TestingRealtimeStoreFactory>();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
+    const std::map<std::string, std::string> partition = {{"dt", "2026-08-02"}};
+
+    ASSERT_OK_AND_ASSIGN(
+        RealtimeStoreState first_state,
+        GetOrCreatePrimaryKeyStore(context, partition, /*bucket=*/0,
+                                   /*restore_max_sequence_number=*/4, GetDefaultPool()));
+    ASSERT_EQ(4, first_state.initial_max_sequence_number);
+
+    const RealtimePartitionBucket partition_bucket(partition, /*bucket=*/0);
+    context->AdvanceMaterializedMaxSequenceNumber(partition_bucket, /*max_sequence_number=*/8);
+    ASSERT_OK_AND_ASSIGN(
+        RealtimeStoreState retained_state,
+        GetOrCreatePrimaryKeyStore(context, partition, /*bucket=*/0,
+                                   /*restore_max_sequence_number=*/6, GetDefaultPool()));
+    ASSERT_EQ(first_state.store, retained_state.store);
+    ASSERT_EQ(8, retained_state.initial_max_sequence_number);
+
+    ASSERT_OK_AND_ASSIGN(
+        RealtimeStoreState restored_state,
+        GetOrCreatePrimaryKeyStore(context, partition, /*bucket=*/0,
+                                   /*restore_max_sequence_number=*/10, GetDefaultPool()));
+    ASSERT_EQ(first_state.store, restored_state.store);
+    ASSERT_EQ(10, restored_state.initial_max_sequence_number);
+}
+
 TEST(RealtimeContextTest, TestCommittedProgressIsMonotonicAndSelective) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index 2ebcede82..e33f48bba 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -44,44 +44,13 @@ namespace paimon {
 
 Result<std::shared_ptr<RealtimePrimaryKeyWriter>> RealtimePrimaryKeyWriter::Create(
     const std::map<std::string, std::string>& partition, int32_t bucket,
-    std::unique_ptr<::ArrowSchema> write_schema,
-    const std::vector<std::string>& trimmed_primary_keys,
-    const std::shared_ptr<RealtimeContext>& realtime_context,
+    const std::shared_ptr<arrow::Schema>& write_schema,
+    const std::shared_ptr<RealtimeContextImpl>& realtime_context,
     const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
-    const std::map<std::string, std::string>& options,
-    const std::shared_ptr<MemoryPool>& memory_pool, int64_t restore_max_sequence_number) {
-    ScopeGuard schema_guard([schema = write_schema.get()]() {
-        if (schema && schema->release) {
-            ArrowSchemaRelease(schema);
-        }
-    });
-    if (!realtime_context) {
-        return Status::Invalid("PK real-time context is null");
-    }
-    if (!merge_tree_writer) {
-        return Status::Invalid("PK real-time merge-tree writer is null");
-    }
-    if (!write_schema || !write_schema->release) {
-        return Status::Invalid("PK real-time write schema is null");
-    }
-    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> imported_schema,
-                                      arrow::ImportSchema(write_schema.get()));
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*imported_schema, write_schema.get()));
-    RealtimeStoreCreateRequest request{
-        std::move(write_schema),
-        options,
-        memory_pool,
-        partition,
-        bucket,
-        PrimaryKeyRealtimeStoreCreateConfig{trimmed_primary_keys, restore_max_sequence_number}};
-    schema_guard.Release();
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
-                           RealtimeContextImpl::Cast(realtime_context));
-    PAIMON_ASSIGN_OR_RAISE(RealtimeStoreState store_state,
-                           realtime_context_impl->GetOrCreateRealtimeStore(std::move(request)));
+    const std::shared_ptr<MemoryPool>& memory_pool, const RealtimeStoreState& store_state) {
     return std::shared_ptr<RealtimePrimaryKeyWriter>(
-        new RealtimePrimaryKeyWriter(store_state.store, merge_tree_writer, realtime_context_impl,
-                                     RealtimePartitionBucket(partition, bucket), imported_schema,
+        new RealtimePrimaryKeyWriter(store_state.store, merge_tree_writer, realtime_context,
+                                     RealtimePartitionBucket(partition, bucket), write_schema,
                                      store_state.initial_offset, memory_pool));
 }
 
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.h b/src/paimon/core/realtime/realtime_primary_key_writer.h
index fa057e079..c1e893c85 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.h
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.h
@@ -24,14 +24,11 @@
 #include <memory>
 #include <mutex>
 #include <string>
-#include <vector>
 
 #include "paimon/core/utils/batch_writer.h"
 #include "paimon/realtime/realtime_context.h"
 #include "paimon/realtime/realtime_store.h"
 
-struct ArrowSchema;
-
 namespace arrow {
 class Schema;
 }  // namespace arrow
@@ -40,20 +37,18 @@ namespace paimon {
 
 class MemoryPool;
 class MergeTreeWriter;
-class RealtimeContext;
 class RealtimeContextImpl;
+struct RealtimeStoreState;
 
 /// Primary-key real-time writer backed by an in-memory mutation indexer.
 class RealtimePrimaryKeyWriter final : public BatchWriter {
  public:
     static Result<std::shared_ptr<RealtimePrimaryKeyWriter>> Create(
         const std::map<std::string, std::string>& partition, int32_t bucket,
-        std::unique_ptr<::ArrowSchema> write_schema,
-        const std::vector<std::string>& trimmed_primary_keys,
-        const std::shared_ptr<RealtimeContext>& realtime_context,
+        const std::shared_ptr<arrow::Schema>& write_schema,
+        const std::shared_ptr<RealtimeContextImpl>& realtime_context,
         const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
-        const std::map<std::string, std::string>& options,
-        const std::shared_ptr<MemoryPool>& memory_pool, int64_t restore_max_sequence_number);
+        const std::shared_ptr<MemoryPool>& memory_pool, const RealtimeStoreState& store_state);
 
     Status Write(std::unique_ptr<RecordBatch>&& batch) override;
     Result<CommitIncrement> PrepareCommit(bool wait_compaction) override;
diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp
index b12e59a84..92155de3b 100644
--- a/src/paimon/core/table/source/table_scan.cpp
+++ b/src/paimon/core/table/source/table_scan.cpp
@@ -41,7 +41,6 @@
 #include "paimon/core/operation/data_evolution_file_store_scan.h"
 #include "paimon/core/operation/file_store_scan.h"
 #include "paimon/core/operation/key_value_file_store_scan.h"
-#include "paimon/core/realtime/primary_key_realtime_options.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
 #include "paimon/core/schema/schema_manager.h"
 #include "paimon/core/schema/schema_validation.h"
@@ -64,6 +63,7 @@
 #include "paimon/core/utils/field_mapping.h"
 #include "paimon/core/utils/file_store_path_factory.h"
 #include "paimon/core/utils/index_file_path_factories.h"
+#include "paimon/core/utils/primary_key_table_utils.h"
 #include "paimon/core/utils/snapshot_manager.h"
 #include "paimon/format/file_format.h"
 #include "paimon/realtime/realtime_context.h"
@@ -233,7 +233,7 @@ Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions&
         return Status::Invalid("real-time union read does not support data evolution");
     }
     if (!table_schema.PrimaryKeys().empty()) {
-        PAIMON_RETURN_NOT_OK(ValidatePrimaryKeyRealtimeOptions(core_options));
+        PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(core_options));
     }
     if (context.IsStreamingMode()) {
         return Status::Invalid("real-time union read currently supports batch scans only");
diff --git a/src/paimon/core/utils/primary_key_table_utils.cpp b/src/paimon/core/utils/primary_key_table_utils.cpp
index cf72da4ae..823d48c41 100644
--- a/src/paimon/core/utils/primary_key_table_utils.cpp
+++ b/src/paimon/core/utils/primary_key_table_utils.cpp
@@ -96,4 +96,36 @@ Result<std::unique_ptr<FieldsComparator>> PrimaryKeyTableUtils::CreateSequenceFi
                                     options.SequenceFieldSortOrderIsAscending());
 }
 
+Status PrimaryKeyTableUtils::ValidateRealtimeOptions(const CoreOptions& options) {
+    if (options.GetBucket() <= 0) {
+        return Status::NotImplemented("PK realtime v1 requires fixed buckets");
+    }
+    if (options.GetMergeEngine() != MergeEngine::DEDUPLICATE) {
+        return Status::NotImplemented("PK realtime v1 supports only the DEDUPLICATE merge engine");
+    }
+    if (options.DataEvolutionEnabled()) {
+        return Status::NotImplemented("PK realtime v1 does not support data evolution");
+    }
+    if (!options.GetFieldsSequenceGroups().empty()) {
+        return Status::NotImplemented("PK realtime v1 does not support sequence groups");
+    }
+    if (options.IgnoreDelete() || options.PartialUpdateRemoveRecordOnDelete() ||
+        options.AggregationRemoveRecordOnDelete() ||
+        !options.GetPartialUpdateRemoveRecordOnSequenceGroup().empty()) {
+        return Status::NotImplemented("PK realtime v1 requires default delete behavior");
+    }
+    if (!options.GetSequenceField().empty()) {
+        return Status::NotImplemented("PK realtime v1 does not support sequence.field");
+    }
+    if (!options.SequenceFieldSortOrderIsAscending()) {
+        return Status::NotImplemented(
+            "PK realtime v1 supports only ascending sequence.field.sort-order");
+    }
+    if (options.NeedLookup() || options.DeletionVectorsEnabled() ||
+        options.GetChangelogProducer() != ChangelogProducer::NONE) {
+        return Status::NotImplemented("PK realtime v1 does not support lookup or early MOR");
+    }
+    return Status::OK();
+}
+
 }  // namespace paimon
diff --git a/src/paimon/core/utils/primary_key_table_utils.h b/src/paimon/core/utils/primary_key_table_utils.h
index 82a108ab7..7877ee4ab 100644
--- a/src/paimon/core/utils/primary_key_table_utils.h
+++ b/src/paimon/core/utils/primary_key_table_utils.h
@@ -24,6 +24,7 @@
 
 #include "arrow/type.h"
 #include "paimon/result.h"
+#include "paimon/status.h"
 
 namespace arrow {
 class Schema;
@@ -57,6 +58,8 @@ class PrimaryKeyTableUtils {
 
     static Result<std::unique_ptr<FieldsComparator>> CreateSequenceFieldsComparator(
         const std::vector<DataField>& value_fields, const CoreOptions& options);
+
+    static Status ValidateRealtimeOptions(const CoreOptions& options);
 };
 
 }  // namespace paimon
diff --git a/src/paimon/core/utils/primary_key_table_utils_test.cpp b/src/paimon/core/utils/primary_key_table_utils_test.cpp
index 12713ca5b..072965cff 100644
--- a/src/paimon/core/utils/primary_key_table_utils_test.cpp
+++ b/src/paimon/core/utils/primary_key_table_utils_test.cpp
@@ -22,7 +22,9 @@
 #include <map>
 #include <memory>
 #include <optional>
+#include <string>
 #include <utility>
+#include <vector>
 
 #include "arrow/type.h"
 #include "gtest/gtest.h"
@@ -112,4 +114,28 @@ TEST(PrimaryKeyTableUtilsTest, TestCreateFirstRowMergeFunctionWithIgnoreDelete)
                         "First row merge engine can not accept DELETE/UPDATE_BEFORE records");
 }
 
+TEST(PrimaryKeyTableUtilsTest, TestSupportedRealtimeOptions) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
+    ASSERT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options));
+}
+
+TEST(PrimaryKeyTableUtilsTest, TestUnsupportedRealtimeOptions) {
+    const std::string sequence_group =
+        std::string(Options::FIELDS_PREFIX) + ".value." + Options::SEQUENCE_GROUP;
+    const std::vector<std::map<std::string, std::string>> unsupported_options = {
+        {{Options::BUCKET, "0"}},
+        {{Options::BUCKET, "1"}, {Options::MERGE_ENGINE, "partial-update"}},
+        {{Options::BUCKET, "1"}, {Options::DATA_EVOLUTION_ENABLED, "true"}},
+        {{Options::BUCKET, "1"}, {sequence_group, "seq"}},
+        {{Options::BUCKET, "1"}, {Options::SEQUENCE_FIELD, "seq"}},
+        {{Options::BUCKET, "1"}, {Options::FORCE_LOOKUP, "true"}},
+        {{Options::BUCKET, "1"}, {Options::DELETION_VECTORS_ENABLED, "true"}},
+        {{Options::BUCKET, "1"}, {Options::CHANGELOG_PRODUCER, "input"}},
+    };
+    for (const std::map<std::string, std::string>& option_map : unsupported_options) {
+        ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
+        ASSERT_NOK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options));
+    }
+}
+
 }  // namespace paimon::test

From 7f4b60b6e9ad8ea165845c72f89074195e51fe23 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Thu, 20 Aug 2026 14:55:20 +0800
Subject: [PATCH 05/62] fix(read): close PK realtime query readers

---
 .../table/source/key_value_table_read.cpp     |   9 +-
 test/inte/realtime_write_inte_test.cpp        | 138 ++++++++++++++++++
 2 files changed, 146 insertions(+), 1 deletion(-)

diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index 770caf1ca..59041b78e 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -65,6 +65,10 @@ class QueryBatchKeyValueReader final : public KeyValueRecordReader {
           value_schema_(value_schema),
           pool_(pool) {}
 
+    ~QueryBatchKeyValueReader() override {
+        Close();
+    }
+
     Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatch() override;
     std::shared_ptr<Metrics> GetReaderMetrics() const override;
     void Close() override;
@@ -161,7 +165,10 @@ void QueryBatchKeyValueReader::Close() {
     row_kinds_.reset();
     key_context_.reset();
     value_context_.reset();
-    reader_->Close();
+    if (reader_) {
+        reader_->Close();
+        reader_.reset();
+    }
 }
 
 Result<std::vector<AdditionalKeyValueReader>> CreateMemoryReaders(
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index f18c3f1e4..cee96301f 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -324,6 +324,101 @@ class QueryTrackingRealtimeStoreFactory final : public RealtimeStoreFactory {
     std::shared_ptr<std::weak_ptr<RealtimeReadView>> query_view_;
 };
 
+class CloseTrackingBatchReader final : public BatchReader {
+ public:
+    CloseTrackingBatchReader(std::unique_ptr<BatchReader> delegate,
+                             const std::shared_ptr<std::atomic<int32_t>>& close_count)
+        : delegate_(std::move(delegate)), close_count_(close_count) {}
+
+    Result<ReadBatch> NextBatch() override {
+        return delegate_->NextBatch();
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return delegate_->GetReaderMetrics();
+    }
+
+    void Close() override {
+        close_count_->fetch_add(1, std::memory_order_release);
+        delegate_->Close();
+    }
+
+ private:
+    std::unique_ptr<BatchReader> delegate_;
+    std::shared_ptr<std::atomic<int32_t>> close_count_;
+};
+
+class CloseTrackingRealtimeStore final : public RealtimeStore {
+ public:
+    CloseTrackingRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
+                               const std::shared_ptr<std::atomic<int32_t>>& close_count,
+                               const std::shared_ptr<std::atomic<bool>>& append_null_reader)
+        : delegate_(delegate), close_count_(close_count), append_null_reader_(append_null_reader) {}
+
+    Status Write(RealtimeWriteBatch&& batch) override {
+        return delegate_->Write(std::move(batch));
+    }
+
+    Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>> SealForCommit() override {
+        return delegate_->SealForCommit();
+    }
+
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
+        const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
+        return delegate_->CreateCommitReaders(segment);
+    }
+
+    Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
+        return delegate_->AcquireReadView();
+    }
+
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
+        const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
+        const RealtimeQueryContext& context) override {
+        PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
+                               delegate_->CreateQueryReaders(view, offset_begin, context));
+        for (std::unique_ptr<BatchReader>& reader : readers) {
+            reader = std::make_unique<CloseTrackingBatchReader>(std::move(reader), close_count_);
+        }
+        if (append_null_reader_->load(std::memory_order_acquire)) {
+            readers.push_back(nullptr);
+        }
+        return readers;
+    }
+
+    Status AdvanceCommittedOffset(int64_t committed_offset) override {
+        return delegate_->AdvanceCommittedOffset(committed_offset);
+    }
+
+    uint64_t GetMemoryUsage() const override {
+        return delegate_->GetMemoryUsage();
+    }
+
+ private:
+    std::shared_ptr<RealtimeStore> delegate_;
+    std::shared_ptr<std::atomic<int32_t>> close_count_;
+    std::shared_ptr<std::atomic<bool>> append_null_reader_;
+};
+
+class CloseTrackingRealtimeStoreFactory final : public RealtimeStoreFactory {
+ public:
+    CloseTrackingRealtimeStoreFactory(const std::shared_ptr<std::atomic<int32_t>>& close_count,
+                                      const std::shared_ptr<std::atomic<bool>>& append_null_reader)
+        : close_count_(close_count), append_null_reader_(append_null_reader) {}
+
+    Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> delegate,
+                               delegate_.Create(std::move(request)));
+        return std::shared_ptr<RealtimeStore>(std::make_shared<CloseTrackingRealtimeStore>(
+            delegate, close_count_, append_null_reader_));
+    }
+
+ private:
+    ArrowRealtimeStoreFactory delegate_;
+    std::shared_ptr<std::atomic<int32_t>> close_count_;
+    std::shared_ptr<std::atomic<bool>> append_null_reader_;
+};
+
 class InvalidReaderRealtimeStore final : public RealtimeStore {
  public:
     explicit InvalidReaderRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate)
@@ -1632,6 +1727,49 @@ TEST_F(RealtimeWriteInteTest, TestPkPluginContract) {
     ASSERT_OK(writer->Close());
 }
 
+TEST_F(RealtimeWriteInteTest, TestPkPluginQueryReaderCloseLifecycle) {
+    CreatePkTable();
+    auto close_count = std::make_shared<std::atomic<int32_t>>(0);
+    auto append_null_reader = std::make_shared<std::atomic<bool>>(false);
+    auto factory =
+        std::make_shared<CloseTrackingRealtimeStoreFactory>(close_count, append_null_reader);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create(factory));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         MakeBatch({Row{1, "one", "p0"}}, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(batch)));
+
+    auto create_reader = [&]() -> Result<std::unique_ptr<BatchReader>> {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Plan> plan,
+                               CreatePlan(realtime_context, /*predicate=*/nullptr));
+        ReadContextBuilder read_builder(table_path_);
+        read_builder.SetOptions(options_)
+            .SetReadFieldNames({"id", "payload", "pt"})
+            .WithRealtimeContext(realtime_context)
+            .WithMemoryPool(pool_);
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableRead> table_read,
+                               TableRead::Create(std::move(read_context)));
+        return table_read->CreateReader(plan->Splits());
+    };
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> explicitly_closed_reader, create_reader());
+    explicitly_closed_reader->Close();
+    explicitly_closed_reader.reset();
+    ASSERT_EQ(1, close_count->load(std::memory_order_acquire));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> destroyed_reader, create_reader());
+    destroyed_reader.reset();
+    ASSERT_EQ(2, close_count->load(std::memory_order_acquire));
+
+    append_null_reader->store(true, std::memory_order_release);
+    ASSERT_NOK_WITH_MSG(create_reader(), "PK real-time store returned a null query reader");
+    ASSERT_EQ(3, close_count->load(std::memory_order_acquire));
+    ASSERT_OK(writer->Close());
+}
+
 TEST_F(RealtimeWriteInteTest, TestRollingFilesPreserveProgress) {
     options_[Options::TARGET_FILE_ROW_NUM] = "10";
     CreateTable(/*partition_keys=*/{});

From 3dee339038f793f079dfb0af03a0e7117f35f927 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Thu, 20 Aug 2026 15:41:07 +0800
Subject: [PATCH 06/62] fix(realtime): close rejected plugin readers

---
 .../realtime/realtime_primary_key_writer.cpp  |   7 +
 .../table/source/key_value_table_read.cpp     |   7 +
 test/inte/realtime_write_inte_test.cpp        | 132 ++++++++++++++----
 3 files changed, 119 insertions(+), 27 deletions(-)

diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index e33f48bba..65bcebcad 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -117,6 +117,13 @@ Status RealtimePrimaryKeyWriter::FlushSegment(
     const std::shared_ptr<RealtimeSegmentHandle>& segment) {
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
                            realtime_store_->CreateCommitReaders(segment));
+    ScopeGuard readers_guard([&readers]() {
+        for (const std::unique_ptr<BatchReader>& reader : readers) {
+            if (reader) {
+                reader->Close();
+            }
+        }
+    });
     for (const std::unique_ptr<BatchReader>& reader : readers) {
         if (!reader) {
             return Status::Invalid("PK real-time store returned a null commit reader");
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index 59041b78e..9b5f6ee83 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -190,6 +190,13 @@ Result<std::vector<AdditionalKeyValueReader>> CreateMemoryReaders(
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> batch_readers,
                            memory.store->CreateQueryReaders(
                                memory.read_view, split->CommittedEndOffset(), query_context));
+    ScopeGuard reader_guard([&batch_readers]() {
+        for (const std::unique_ptr<BatchReader>& reader : batch_readers) {
+            if (reader) {
+                reader->Close();
+            }
+        }
+    });
     if (batch_readers.empty()) {
         return Status::Invalid("PK real-time store returned no query readers for active memory");
     }
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index cee96301f..e6000561f 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -348,12 +348,20 @@ class CloseTrackingBatchReader final : public BatchReader {
     std::shared_ptr<std::atomic<int32_t>> close_count_;
 };
 
+struct CloseTrackingReaderState {
+    std::shared_ptr<std::atomic<int32_t>> query_close_count =
+        std::make_shared<std::atomic<int32_t>>(0);
+    std::shared_ptr<std::atomic<int32_t>> commit_close_count =
+        std::make_shared<std::atomic<int32_t>>(0);
+    int32_t query_null_index = -1;
+    int32_t commit_null_index = -1;
+};
+
 class CloseTrackingRealtimeStore final : public RealtimeStore {
  public:
     CloseTrackingRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
-                               const std::shared_ptr<std::atomic<int32_t>>& close_count,
-                               const std::shared_ptr<std::atomic<bool>>& append_null_reader)
-        : delegate_(delegate), close_count_(close_count), append_null_reader_(append_null_reader) {}
+                               const std::shared_ptr<CloseTrackingReaderState>& state)
+        : delegate_(delegate), state_(state) {}
 
     Status Write(RealtimeWriteBatch&& batch) override {
         return delegate_->Write(std::move(batch));
@@ -365,7 +373,14 @@ class CloseTrackingRealtimeStore final : public RealtimeStore {
 
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
         const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
-        return delegate_->CreateCommitReaders(segment);
+        PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
+                               delegate_->CreateCommitReaders(segment));
+        for (std::unique_ptr<BatchReader>& reader : readers) {
+            reader = std::make_unique<CloseTrackingBatchReader>(std::move(reader),
+                                                                state_->commit_close_count);
+        }
+        PAIMON_RETURN_NOT_OK(InsertNullReader(state_->commit_null_index, &readers));
+        return readers;
     }
 
     Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
@@ -378,11 +393,10 @@ class CloseTrackingRealtimeStore final : public RealtimeStore {
         PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
                                delegate_->CreateQueryReaders(view, offset_begin, context));
         for (std::unique_ptr<BatchReader>& reader : readers) {
-            reader = std::make_unique<CloseTrackingBatchReader>(std::move(reader), close_count_);
-        }
-        if (append_null_reader_->load(std::memory_order_acquire)) {
-            readers.push_back(nullptr);
+            reader = std::make_unique<CloseTrackingBatchReader>(std::move(reader),
+                                                                state_->query_close_count);
         }
+        PAIMON_RETURN_NOT_OK(InsertNullReader(state_->query_null_index, &readers));
         return readers;
     }
 
@@ -395,28 +409,38 @@ class CloseTrackingRealtimeStore final : public RealtimeStore {
     }
 
  private:
+    static Status InsertNullReader(int32_t index,
+                                   std::vector<std::unique_ptr<BatchReader>>* readers) {
+        if (index < 0) {
+            return Status::OK();
+        }
+        if (index > static_cast<int32_t>(readers->size())) {
+            return Status::Invalid("null reader index exceeds reader count");
+        }
+        readers->insert(readers->begin() + index, nullptr);
+        return Status::OK();
+    }
+
     std::shared_ptr<RealtimeStore> delegate_;
-    std::shared_ptr<std::atomic<int32_t>> close_count_;
-    std::shared_ptr<std::atomic<bool>> append_null_reader_;
+    std::shared_ptr<CloseTrackingReaderState> state_;
 };
 
 class CloseTrackingRealtimeStoreFactory final : public RealtimeStoreFactory {
  public:
-    CloseTrackingRealtimeStoreFactory(const std::shared_ptr<std::atomic<int32_t>>& close_count,
-                                      const std::shared_ptr<std::atomic<bool>>& append_null_reader)
-        : close_count_(close_count), append_null_reader_(append_null_reader) {}
+    explicit CloseTrackingRealtimeStoreFactory(
+        const std::shared_ptr<CloseTrackingReaderState>& state)
+        : state_(state) {}
 
     Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> delegate,
                                delegate_.Create(std::move(request)));
-        return std::shared_ptr<RealtimeStore>(std::make_shared<CloseTrackingRealtimeStore>(
-            delegate, close_count_, append_null_reader_));
+        return std::shared_ptr<RealtimeStore>(
+            std::make_shared<CloseTrackingRealtimeStore>(delegate, state_));
     }
 
  private:
     ArrowRealtimeStoreFactory delegate_;
-    std::shared_ptr<std::atomic<int32_t>> close_count_;
-    std::shared_ptr<std::atomic<bool>> append_null_reader_;
+    std::shared_ptr<CloseTrackingReaderState> state_;
 };
 
 class InvalidReaderRealtimeStore final : public RealtimeStore {
@@ -1727,12 +1751,10 @@ TEST_F(RealtimeWriteInteTest, TestPkPluginContract) {
     ASSERT_OK(writer->Close());
 }
 
-TEST_F(RealtimeWriteInteTest, TestPkPluginQueryReaderCloseLifecycle) {
+TEST_F(RealtimeWriteInteTest, TestPkQueryReaderClose) {
     CreatePkTable();
-    auto close_count = std::make_shared<std::atomic<int32_t>>(0);
-    auto append_null_reader = std::make_shared<std::atomic<bool>>(false);
-    auto factory =
-        std::make_shared<CloseTrackingRealtimeStoreFactory>(close_count, append_null_reader);
+    auto state = std::make_shared<CloseTrackingReaderState>();
+    auto factory = std::make_shared<CloseTrackingRealtimeStoreFactory>(state);
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                          RealtimeContext::Create(factory));
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
@@ -1758,15 +1780,71 @@ TEST_F(RealtimeWriteInteTest, TestPkPluginQueryReaderCloseLifecycle) {
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> explicitly_closed_reader, create_reader());
     explicitly_closed_reader->Close();
     explicitly_closed_reader.reset();
-    ASSERT_EQ(1, close_count->load(std::memory_order_acquire));
+    ASSERT_EQ(1, state->query_close_count->load(std::memory_order_acquire));
 
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> destroyed_reader, create_reader());
     destroyed_reader.reset();
-    ASSERT_EQ(2, close_count->load(std::memory_order_acquire));
+    ASSERT_EQ(2, state->query_close_count->load(std::memory_order_acquire));
+
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestPkQueryReaderCloseFailure) {
+    CreatePkTable();
+    auto state = std::make_shared<CloseTrackingReaderState>();
+    auto factory = std::make_shared<CloseTrackingRealtimeStoreFactory>(state);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create(factory));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
+                         MakeBatch({Row{1, "one", "p0"}}, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(first_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_EQ(1, progress.size());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
+                         MakeBatch({Row{2, "two", "p0"}}, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(second_batch)));
 
-    append_null_reader->store(true, std::memory_order_release);
-    ASSERT_NOK_WITH_MSG(create_reader(), "PK real-time store returned a null query reader");
-    ASSERT_EQ(3, close_count->load(std::memory_order_acquire));
+    auto create_reader = [&]() -> Result<std::unique_ptr<BatchReader>> {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Plan> plan,
+                               CreatePlan(realtime_context, /*predicate=*/nullptr));
+        ReadContextBuilder read_builder(table_path_);
+        read_builder.SetOptions(options_)
+            .SetReadFieldNames({"id", "payload", "pt"})
+            .WithRealtimeContext(realtime_context)
+            .WithMemoryPool(pool_);
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableRead> table_read,
+                               TableRead::Create(std::move(read_context)));
+        return table_read->CreateReader(plan->Splits());
+    };
+
+    for (int32_t null_index = 0; null_index <= 2; ++null_index) {
+        state->query_null_index = null_index;
+        ASSERT_NOK_WITH_MSG(create_reader(), "PK real-time store returned a null query reader");
+        ASSERT_EQ(2 * (null_index + 1), state->query_close_count->load(std::memory_order_acquire));
+    }
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestPkCommitReaderCloseFailure) {
+    CreatePkTable();
+    auto state = std::make_shared<CloseTrackingReaderState>();
+    state->commit_null_index = 1;
+    auto factory = std::make_shared<CloseTrackingRealtimeStoreFactory>(state);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create(factory));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         MakeBatch({Row{1, "one", "p0"}}, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(batch)));
+
+    ASSERT_NOK_WITH_MSG(writer->PrepareCommitWithProgress(/*commit_identifier=*/0),
+                        "PK real-time store returned a null commit reader");
+    ASSERT_EQ(1, state->commit_close_count->load(std::memory_order_acquire));
     ASSERT_OK(writer->Close());
 }
 

From b36b24f0873ebdb77fbcc6c7792d0129fcf052f6 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Thu, 20 Aug 2026 18:05:54 +0800
Subject: [PATCH 07/62] fix(read): preserve PK reader metrics after close

---
 src/paimon/core/table/source/key_value_table_read.cpp | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index 9b5f6ee83..76160ac93 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -85,6 +85,7 @@ class QueryBatchKeyValueReader final : public KeyValueRecordReader {
     std::shared_ptr<arrow::Int8Array> row_kinds_;
     std::shared_ptr<ColumnarBatchContext> key_context_;
     std::shared_ptr<ColumnarBatchContext> value_context_;
+    bool closed_ = false;
 };
 
 class QueryBatchKeyValueReader::Iterator final : public KeyValueRecordReader::Iterator {
@@ -160,6 +161,10 @@ std::shared_ptr<Metrics> QueryBatchKeyValueReader::GetReaderMetrics() const {
 }
 
 void QueryBatchKeyValueReader::Close() {
+    if (closed_) {
+        return;
+    }
+    closed_ = true;
     values_.reset();
     sequences_.reset();
     row_kinds_.reset();
@@ -167,7 +172,6 @@ void QueryBatchKeyValueReader::Close() {
     value_context_.reset();
     if (reader_) {
         reader_->Close();
-        reader_.reset();
     }
 }
 

From 20b834ea7bbc24be751546f7349a71ea9b5d97c8 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Thu, 20 Aug 2026 18:06:13 +0800
Subject: [PATCH 08/62] refactor(realtime): colocate PK realtime option
 validation

---
 .../core/operation/file_store_write.cpp       |  3 +-
 .../realtime/primary_key_realtime_store.cpp   | 34 +++++++++++++++++++
 .../realtime/primary_key_realtime_store.h     |  3 ++
 .../primary_key_realtime_store_test.cpp       | 26 ++++++++++++++
 src/paimon/core/table/source/table_scan.cpp   |  4 +--
 .../core/utils/primary_key_table_utils.cpp    | 32 -----------------
 .../core/utils/primary_key_table_utils.h      |  2 --
 .../utils/primary_key_table_utils_test.cpp    | 25 --------------
 8 files changed, 67 insertions(+), 62 deletions(-)

diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp
index f216476bd..4d4f45156 100644
--- a/src/paimon/core/operation/file_store_write.cpp
+++ b/src/paimon/core/operation/file_store_write.cpp
@@ -36,6 +36,7 @@
 #include "paimon/core/operation/key_value_file_store_write.h"
 #include "paimon/core/options/merge_engine.h"
 #include "paimon/core/postpone/postpone_bucket_file_store_write.h"
+#include "paimon/core/realtime/primary_key_realtime_store.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
 #include "paimon/core/schema/schema_manager.h"
 #include "paimon/core/schema/table_schema.h"
@@ -197,7 +198,7 @@ Result<std::unique_ptr<FileStoreWrite>> FileStoreWrite::Create(std::unique_ptr<W
     } else {
         // pk table
         if (ctx->GetRealtimeContext()) {
-            PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options));
+            PAIMON_RETURN_NOT_OK(ValidatePrimaryKeyRealtimeOptions(options));
             if (ignore_previous_files) {
                 return Status::NotImplemented(
                     "PK realtime v1 requires restore from the latest snapshot");
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index 84afb97a4..afdc0c73c 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -36,6 +36,7 @@
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/fields_comparator.h"
+#include "paimon/core/core_options.h"
 #include "paimon/core/io/key_value_in_memory_record_reader.h"
 #include "paimon/core/io/key_value_projection_consumer.h"
 #include "paimon/core/io/key_value_projection_reader.h"
@@ -45,6 +46,39 @@
 #include "paimon/macros.h"
 
 namespace paimon {
+
+Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options) {
+    if (options.GetBucket() <= 0) {
+        return Status::NotImplemented("PK realtime v1 requires fixed buckets");
+    }
+    if (options.GetMergeEngine() != MergeEngine::DEDUPLICATE) {
+        return Status::NotImplemented("PK realtime v1 supports only the DEDUPLICATE merge engine");
+    }
+    if (options.DataEvolutionEnabled()) {
+        return Status::NotImplemented("PK realtime v1 does not support data evolution");
+    }
+    if (!options.GetFieldsSequenceGroups().empty()) {
+        return Status::NotImplemented("PK realtime v1 does not support sequence groups");
+    }
+    if (options.IgnoreDelete() || options.PartialUpdateRemoveRecordOnDelete() ||
+        options.AggregationRemoveRecordOnDelete() ||
+        !options.GetPartialUpdateRemoveRecordOnSequenceGroup().empty()) {
+        return Status::NotImplemented("PK realtime v1 requires default delete behavior");
+    }
+    if (!options.GetSequenceField().empty()) {
+        return Status::NotImplemented("PK realtime v1 does not support sequence.field");
+    }
+    if (!options.SequenceFieldSortOrderIsAscending()) {
+        return Status::NotImplemented(
+            "PK realtime v1 supports only ascending sequence.field.sort-order");
+    }
+    if (options.NeedLookup() || options.DeletionVectorsEnabled() ||
+        options.GetChangelogProducer() != ChangelogProducer::NONE) {
+        return Status::NotImplemented("PK realtime v1 does not support lookup or early MOR");
+    }
+    return Status::OK();
+}
+
 namespace {
 
 uint64_t GetArrayMemoryUsage(const std::shared_ptr<arrow::ArrayData>& data) {
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.h b/src/paimon/core/realtime/primary_key_realtime_store.h
index 05225ed19..017864c04 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.h
+++ b/src/paimon/core/realtime/primary_key_realtime_store.h
@@ -33,6 +33,7 @@ class Schema;
 
 namespace paimon {
 
+class CoreOptions;
 class FieldsComparator;
 struct KeyValue;
 class MemoryPool;
@@ -40,6 +41,8 @@ class InternalRow;
 template <typename T>
 class MergeFunctionWrapper;
 
+Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options);
+
 /// Optional metadata exposed by PK query readers with a known inclusive key range.
 class PrimaryKeyRangeProvider {
  public:
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index 9da272e0f..cbbf9c82a 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -19,6 +19,7 @@
 
 #include "paimon/core/realtime/primary_key_realtime_store.h"
 
+#include <map>
 #include <memory>
 #include <string>
 #include <vector>
@@ -30,6 +31,7 @@
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/fields_comparator.h"
+#include "paimon/core/core_options.h"
 #include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
 #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
 #include "paimon/memory/memory_pool.h"
@@ -37,6 +39,30 @@
 
 namespace paimon::test {
 
+TEST(PrimaryKeyRealtimeStoreOptionsTest, TestSupportedOptions) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
+    ASSERT_OK(ValidatePrimaryKeyRealtimeOptions(options));
+}
+
+TEST(PrimaryKeyRealtimeStoreOptionsTest, TestUnsupportedOptions) {
+    const std::string sequence_group =
+        std::string(Options::FIELDS_PREFIX) + ".value." + Options::SEQUENCE_GROUP;
+    const std::vector<std::map<std::string, std::string>> unsupported_options = {
+        {{Options::BUCKET, "0"}},
+        {{Options::BUCKET, "1"}, {Options::MERGE_ENGINE, "partial-update"}},
+        {{Options::BUCKET, "1"}, {Options::DATA_EVOLUTION_ENABLED, "true"}},
+        {{Options::BUCKET, "1"}, {sequence_group, "seq"}},
+        {{Options::BUCKET, "1"}, {Options::SEQUENCE_FIELD, "seq"}},
+        {{Options::BUCKET, "1"}, {Options::FORCE_LOOKUP, "true"}},
+        {{Options::BUCKET, "1"}, {Options::DELETION_VECTORS_ENABLED, "true"}},
+        {{Options::BUCKET, "1"}, {Options::CHANGELOG_PRODUCER, "input"}},
+    };
+    for (const std::map<std::string, std::string>& option_map : unsupported_options) {
+        ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
+        ASSERT_NOK(ValidatePrimaryKeyRealtimeOptions(options));
+    }
+}
+
 class PrimaryKeyRealtimeStoreTest : public testing::Test {
  public:
     void SetUp() override {
diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp
index 92155de3b..dcf10e90c 100644
--- a/src/paimon/core/table/source/table_scan.cpp
+++ b/src/paimon/core/table/source/table_scan.cpp
@@ -41,6 +41,7 @@
 #include "paimon/core/operation/data_evolution_file_store_scan.h"
 #include "paimon/core/operation/file_store_scan.h"
 #include "paimon/core/operation/key_value_file_store_scan.h"
+#include "paimon/core/realtime/primary_key_realtime_store.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
 #include "paimon/core/schema/schema_manager.h"
 #include "paimon/core/schema/schema_validation.h"
@@ -63,7 +64,6 @@
 #include "paimon/core/utils/field_mapping.h"
 #include "paimon/core/utils/file_store_path_factory.h"
 #include "paimon/core/utils/index_file_path_factories.h"
-#include "paimon/core/utils/primary_key_table_utils.h"
 #include "paimon/core/utils/snapshot_manager.h"
 #include "paimon/format/file_format.h"
 #include "paimon/realtime/realtime_context.h"
@@ -233,7 +233,7 @@ Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions&
         return Status::Invalid("real-time union read does not support data evolution");
     }
     if (!table_schema.PrimaryKeys().empty()) {
-        PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(core_options));
+        PAIMON_RETURN_NOT_OK(ValidatePrimaryKeyRealtimeOptions(core_options));
     }
     if (context.IsStreamingMode()) {
         return Status::Invalid("real-time union read currently supports batch scans only");
diff --git a/src/paimon/core/utils/primary_key_table_utils.cpp b/src/paimon/core/utils/primary_key_table_utils.cpp
index 823d48c41..cf72da4ae 100644
--- a/src/paimon/core/utils/primary_key_table_utils.cpp
+++ b/src/paimon/core/utils/primary_key_table_utils.cpp
@@ -96,36 +96,4 @@ Result<std::unique_ptr<FieldsComparator>> PrimaryKeyTableUtils::CreateSequenceFi
                                     options.SequenceFieldSortOrderIsAscending());
 }
 
-Status PrimaryKeyTableUtils::ValidateRealtimeOptions(const CoreOptions& options) {
-    if (options.GetBucket() <= 0) {
-        return Status::NotImplemented("PK realtime v1 requires fixed buckets");
-    }
-    if (options.GetMergeEngine() != MergeEngine::DEDUPLICATE) {
-        return Status::NotImplemented("PK realtime v1 supports only the DEDUPLICATE merge engine");
-    }
-    if (options.DataEvolutionEnabled()) {
-        return Status::NotImplemented("PK realtime v1 does not support data evolution");
-    }
-    if (!options.GetFieldsSequenceGroups().empty()) {
-        return Status::NotImplemented("PK realtime v1 does not support sequence groups");
-    }
-    if (options.IgnoreDelete() || options.PartialUpdateRemoveRecordOnDelete() ||
-        options.AggregationRemoveRecordOnDelete() ||
-        !options.GetPartialUpdateRemoveRecordOnSequenceGroup().empty()) {
-        return Status::NotImplemented("PK realtime v1 requires default delete behavior");
-    }
-    if (!options.GetSequenceField().empty()) {
-        return Status::NotImplemented("PK realtime v1 does not support sequence.field");
-    }
-    if (!options.SequenceFieldSortOrderIsAscending()) {
-        return Status::NotImplemented(
-            "PK realtime v1 supports only ascending sequence.field.sort-order");
-    }
-    if (options.NeedLookup() || options.DeletionVectorsEnabled() ||
-        options.GetChangelogProducer() != ChangelogProducer::NONE) {
-        return Status::NotImplemented("PK realtime v1 does not support lookup or early MOR");
-    }
-    return Status::OK();
-}
-
 }  // namespace paimon
diff --git a/src/paimon/core/utils/primary_key_table_utils.h b/src/paimon/core/utils/primary_key_table_utils.h
index 7877ee4ab..c40e92cda 100644
--- a/src/paimon/core/utils/primary_key_table_utils.h
+++ b/src/paimon/core/utils/primary_key_table_utils.h
@@ -58,8 +58,6 @@ class PrimaryKeyTableUtils {
 
     static Result<std::unique_ptr<FieldsComparator>> CreateSequenceFieldsComparator(
         const std::vector<DataField>& value_fields, const CoreOptions& options);
-
-    static Status ValidateRealtimeOptions(const CoreOptions& options);
 };
 
 }  // namespace paimon
diff --git a/src/paimon/core/utils/primary_key_table_utils_test.cpp b/src/paimon/core/utils/primary_key_table_utils_test.cpp
index 072965cff..1a7345fdf 100644
--- a/src/paimon/core/utils/primary_key_table_utils_test.cpp
+++ b/src/paimon/core/utils/primary_key_table_utils_test.cpp
@@ -19,7 +19,6 @@
 #include "paimon/core/utils/primary_key_table_utils.h"
 
 #include <cstdint>
-#include <map>
 #include <memory>
 #include <optional>
 #include <string>
@@ -114,28 +113,4 @@ TEST(PrimaryKeyTableUtilsTest, TestCreateFirstRowMergeFunctionWithIgnoreDelete)
                         "First row merge engine can not accept DELETE/UPDATE_BEFORE records");
 }
 
-TEST(PrimaryKeyTableUtilsTest, TestSupportedRealtimeOptions) {
-    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
-    ASSERT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options));
-}
-
-TEST(PrimaryKeyTableUtilsTest, TestUnsupportedRealtimeOptions) {
-    const std::string sequence_group =
-        std::string(Options::FIELDS_PREFIX) + ".value." + Options::SEQUENCE_GROUP;
-    const std::vector<std::map<std::string, std::string>> unsupported_options = {
-        {{Options::BUCKET, "0"}},
-        {{Options::BUCKET, "1"}, {Options::MERGE_ENGINE, "partial-update"}},
-        {{Options::BUCKET, "1"}, {Options::DATA_EVOLUTION_ENABLED, "true"}},
-        {{Options::BUCKET, "1"}, {sequence_group, "seq"}},
-        {{Options::BUCKET, "1"}, {Options::SEQUENCE_FIELD, "seq"}},
-        {{Options::BUCKET, "1"}, {Options::FORCE_LOOKUP, "true"}},
-        {{Options::BUCKET, "1"}, {Options::DELETION_VECTORS_ENABLED, "true"}},
-        {{Options::BUCKET, "1"}, {Options::CHANGELOG_PRODUCER, "input"}},
-    };
-    for (const std::map<std::string, std::string>& option_map : unsupported_options) {
-        ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
-        ASSERT_NOK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options));
-    }
-}
-
 }  // namespace paimon::test

From b89a9fc44380283ce5d7c5e415042f3c301370da Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Thu, 20 Aug 2026 20:45:39 +0800
Subject: [PATCH 09/62] test(realtime): improve primary key coverage

---
 .../primary_key_realtime_store_test.cpp       | 281 ++++++++++++++----
 test/inte/realtime_write_inte_test.cpp        | 248 +++++++++++++++-
 2 files changed, 463 insertions(+), 66 deletions(-)

diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index cbbf9c82a..5c04d4310 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -19,9 +19,13 @@
 
 #include "paimon/core/realtime/primary_key_realtime_store.h"
 
+#include <cstdint>
+#include <limits>
 #include <map>
 #include <memory>
+#include <optional>
 #include <string>
+#include <utility>
 #include <vector>
 
 #include "arrow/api.h"
@@ -29,7 +33,6 @@
 #include "arrow/ipc/json_simple.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
-#include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/fields_comparator.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
@@ -69,24 +72,37 @@ class PrimaryKeyRealtimeStoreTest : public testing::Test {
         pool_ = std::shared_ptr<MemoryPool>(GetMemoryPool());
         schema_ = arrow::schema(
             {arrow::field("id", arrow::int64()), arrow::field("value", arrow::utf8())});
-        ASSERT_OK_AND_ASSIGN(key_comparator_,
-                             FieldsComparator::Create({DataField(0, schema_->field(0))},
-                                                      /*is_ascending_order=*/true));
+        ASSERT_OK_AND_ASSIGN(store_, CreateStore(schema_, {"id"}, /*restore_max_sequence=*/4));
+    }
+
+    Result<std::shared_ptr<PrimaryKeyRealtimeStore>> CreateStore(
+        const std::shared_ptr<arrow::Schema>& schema, const std::vector<std::string>& primary_keys,
+        int64_t restore_max_sequence) const {
+        std::vector<DataField> key_fields;
+        key_fields.reserve(primary_keys.size());
+        for (const std::string& primary_key : primary_keys) {
+            const int32_t index = schema->GetFieldIndex(primary_key);
+            key_fields.emplace_back(index, schema->field(index));
+        }
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FieldsComparator> key_comparator,
+                               FieldsComparator::Create(key_fields,
+                                                        /*is_ascending_order=*/true));
         auto merge_factory = []() {
             auto merge_function =
                 std::make_unique<DeduplicateMergeFunction>(/*ignore_delete=*/false);
             return std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge_function));
         };
-        ASSERT_OK_AND_ASSIGN(
-            store_, PrimaryKeyRealtimeStore::Create(schema_, {"id"}, key_comparator_, merge_factory,
-                                                    /*restore_max_sequence_number=*/4,
-                                                    /*read_batch_size=*/1024, pool_));
+        return PrimaryKeyRealtimeStore::Create(schema, primary_keys, key_comparator, merge_factory,
+                                               restore_max_sequence,
+                                               /*read_batch_size=*/2, pool_);
     }
 
     std::unique_ptr<RecordBatch> MakeBatch(
-        const std::string& json, const std::vector<RecordBatch::RowKind>& row_kinds = {}) const {
+        const std::string& json, const std::vector<RecordBatch::RowKind>& row_kinds = {},
+        const std::shared_ptr<arrow::Schema>& schema = nullptr) const {
+        const std::shared_ptr<arrow::Schema>& batch_schema = schema ? schema : schema_;
         std::shared_ptr<arrow::Array> array =
-            arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema_->fields()), json)
+            arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(batch_schema->fields()), json)
                 .ValueOrDie();
         ArrowArray c_array;
         EXPECT_TRUE(arrow::ExportArray(*array, &c_array).ok());
@@ -95,35 +111,39 @@ class PrimaryKeyRealtimeStoreTest : public testing::Test {
         return builder.Finish().value();
     }
 
-    std::unique_ptr<ArrowSchema> MakeReadSchema(bool include_sequence) const {
-        arrow::FieldVector fields;
-        if (include_sequence) {
-            fields.push_back(
-                DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber()));
-        }
-        fields.insert(fields.end(), schema_->fields().begin(), schema_->fields().end());
+    std::unique_ptr<ArrowSchema> MakeReadSchema(const arrow::FieldVector& fields) const {
         auto c_schema = std::make_unique<ArrowSchema>();
         EXPECT_TRUE(arrow::ExportSchema(*arrow::schema(fields), c_schema.get()).ok());
         return c_schema;
     }
 
-    void AssertReaderOutput(BatchReader* reader, const std::shared_ptr<arrow::DataType>& type,
+    void AssertReaderOutput(const std::vector<std::unique_ptr<BatchReader>>& readers,
+                            const std::shared_ptr<arrow::DataType>& type,
                             const std::string& json) const {
-        ASSERT_NE(nullptr, reader);
-        ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch());
-        ASSERT_FALSE(BatchReader::IsEofBatch(batch));
-        arrow::Result<std::shared_ptr<arrow::Array>> imported_result =
-            arrow::ImportArray(batch.first.get(), batch.second.get());
-        ASSERT_TRUE(imported_result.ok()) << imported_result.status().ToString();
-        std::shared_ptr<arrow::Array> actual = std::move(imported_result).ValueOrDie();
+        std::vector<std::shared_ptr<arrow::Array>> batches;
+        for (const std::unique_ptr<BatchReader>& reader : readers) {
+            while (true) {
+                ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch());
+                if (BatchReader::IsEofBatch(batch)) {
+                    break;
+                }
+                arrow::Result<std::shared_ptr<arrow::Array>> imported =
+                    arrow::ImportArray(batch.first.get(), batch.second.get());
+                ASSERT_TRUE(imported.ok()) << imported.status().ToString();
+                batches.push_back(std::move(imported).ValueOrDie());
+            }
+        }
+        ASSERT_FALSE(batches.empty());
+        arrow::Result<std::shared_ptr<arrow::Array>> concatenated = arrow::Concatenate(batches);
+        ASSERT_TRUE(concatenated.ok()) << concatenated.status().ToString();
+        std::shared_ptr<arrow::Array> actual = std::move(concatenated).ValueOrDie();
         std::shared_ptr<arrow::Array> expected =
             arrow::ipc::internal::json::ArrayFromJSON(type, json).ValueOrDie();
         ASSERT_TRUE(actual->Equals(*expected))
             << "expected: " << expected->ToString() << ", actual: " << actual->ToString();
-
-        ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch eof, reader->NextBatch());
-        ASSERT_TRUE(BatchReader::IsEofBatch(eof));
-        reader->Close();
+        for (const std::unique_ptr<BatchReader>& reader : readers) {
+            reader->Close();
+        }
     }
 
     std::shared_ptr<arrow::DataType> CommitType() const {
@@ -143,10 +163,18 @@ class PrimaryKeyRealtimeStoreTest : public testing::Test {
         });
     }
 
+    arrow::FieldVector FullQueryFields(
+        const std::shared_ptr<arrow::Schema>& schema = nullptr) const {
+        const std::shared_ptr<arrow::Schema>& query_schema = schema ? schema : schema_;
+        arrow::FieldVector fields = {
+            DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())};
+        fields.insert(fields.end(), query_schema->fields().begin(), query_schema->fields().end());
+        return fields;
+    }
+
  protected:
     std::shared_ptr<MemoryPool> pool_;
     std::shared_ptr<arrow::Schema> schema_;
-    std::shared_ptr<FieldsComparator> key_comparator_;
     std::shared_ptr<PrimaryKeyRealtimeStore> store_;
 };
 
@@ -172,30 +200,48 @@ TEST_F(PrimaryKeyRealtimeStoreTest, TestWriteAndSeal) {
     ASSERT_EQ(OffsetRange(0, 3), segment.value()->GetOffsetRange());
     ASSERT_GT(store_->GetMemoryUsage(), 0);
 
-    auto merge_factory = []() {
-        auto merge_function = std::make_unique<DeduplicateMergeFunction>(/*ignore_delete=*/false);
-        return std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge_function));
+    struct ValidationCase {
+        int64_t restore_max_sequence;
+        std::string error;
+    };
+    const std::vector<ValidationCase> cases = {
+        {-2, "restore max sequence number must be at least -1"},
+        {std::numeric_limits<int64_t>::max(), "sequence number has reached INT64_MAX"},
     };
-    ASSERT_NOK_WITH_MSG(PrimaryKeyRealtimeStore::Create(
-                            schema_, {"id"}, key_comparator_, merge_factory,
-                            /*restore_max_sequence_number=*/-2, /*read_batch_size=*/1024, pool_),
-                        "restore max sequence number must be at least -1");
+    for (const ValidationCase& test_case : cases) {
+        ASSERT_NOK_WITH_MSG(CreateStore(schema_, {"id"}, test_case.restore_max_sequence),
+                            test_case.error);
+    }
 }
 
-TEST_F(PrimaryKeyRealtimeStoreTest, TestCommitReaderPreservesMutations) {
+TEST_F(PrimaryKeyRealtimeStoreTest, TestCommitBatches) {
+    ASSERT_OK(store_->Write(RealtimeWriteBatch{
+        MakeBatch(R"([[3, "three"], [1, "before"]])",
+                  {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::UPDATE_BEFORE}),
+        OffsetRange(0, 2)}));
+    ASSERT_OK(store_->Write(RealtimeWriteBatch{
+        MakeBatch(R"([[2, "after"]])", {RecordBatch::RowKind::UPDATE_AFTER}), OffsetRange(2, 3)}));
     ASSERT_OK(store_->Write(
-        RealtimeWriteBatch{MakeBatch(R"([[2, "old"], [1, "one"], [2, "new"]])",
-                                     {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::INSERT,
-                                      RecordBatch::RowKind::UPDATE_AFTER}),
-                           OffsetRange(0, 3)}));
+        RealtimeWriteBatch{MakeBatch(R"([[4, "deleted"], [0, "zero"]])",
+                                     {RecordBatch::RowKind::DELETE, RecordBatch::RowKind::INSERT}),
+                           OffsetRange(3, 5)}));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
                          store_->SealForCommit());
     ASSERT_TRUE(segment.has_value());
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
                          store_->CreateCommitReaders(segment.value()));
-    ASSERT_EQ(1, readers.size());
-    AssertReaderOutput(readers[0].get(), CommitType(),
-                       R"([[0, 2, "old"], [0, 1, "one"], [2, 2, "new"]])");
+    AssertReaderOutput(readers, CommitType(),
+                       R"([[0, 3, "three"], [1, 1, "before"], [2, 2, "after"],
+                            [3, 4, "deleted"], [0, 0, "zero"]])");
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store_->AcquireReadView());
+    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(FullQueryFields());
+    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+    ASSERT_OK_AND_ASSIGN(readers, store_->CreateQueryReaders(view, /*offset_begin=*/0, context));
+    AssertReaderOutput(readers, QueryType(),
+                       R"([[0, 9, 0, "zero"], [1, 6, 1, "before"], [2, 7, 2, "after"],
+                            [0, 5, 3, "three"], [3, 8, 4, "deleted"]])");
 }
 
 TEST_F(PrimaryKeyRealtimeStoreTest, TestMutationMerge) {
@@ -207,13 +253,12 @@ TEST_F(PrimaryKeyRealtimeStoreTest, TestMutationMerge) {
         OffsetRange(2, 4)}));
 
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store_->AcquireReadView());
-    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(/*include_sequence=*/true);
+    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(FullQueryFields());
     RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
                                  /*enable_predicate_pushdown=*/false};
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
                          store_->CreateQueryReaders(view, /*offset_begin=*/0, context));
-    ASSERT_EQ(1, readers.size());
-    AssertReaderOutput(readers[0].get(), QueryType(), R"([[2, 7, 1, "new"], [3, 8, 2, "gone"]])");
+    AssertReaderOutput(readers, QueryType(), R"([[2, 7, 1, "new"], [3, 8, 2, "gone"]])");
 }
 
 TEST_F(PrimaryKeyRealtimeStoreTest, TestReadViewLifecycle) {
@@ -230,15 +275,14 @@ TEST_F(PrimaryKeyRealtimeStoreTest, TestReadViewLifecycle) {
     ASSERT_OK(
         store_->Write(RealtimeWriteBatch{MakeBatch(R"([[13, "later"]])"), OffsetRange(13, 14)}));
 
-    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(/*include_sequence=*/true);
+    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(FullQueryFields());
     RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
                                  /*enable_predicate_pushdown=*/false};
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
                          store_->CreateQueryReaders(view, /*offset_begin=*/11, context));
-    ASSERT_EQ(1, readers.size());
-    AssertReaderOutput(readers[0].get(), QueryType(), R"([[0, 6, 11, "b"], [0, 7, 12, "c"]])");
+    AssertReaderOutput(readers, QueryType(), R"([[0, 6, 11, "b"], [0, 7, 12, "c"]])");
 
-    std::unique_ptr<ArrowSchema> empty_schema = MakeReadSchema(/*include_sequence=*/true);
+    std::unique_ptr<ArrowSchema> empty_schema = MakeReadSchema(FullQueryFields());
     context.read_schema = empty_schema.get();
     ASSERT_OK_AND_ASSIGN(readers, store_->CreateQueryReaders(view, /*offset_begin=*/13, context));
     ASSERT_TRUE(readers.empty());
@@ -251,20 +295,135 @@ TEST_F(PrimaryKeyRealtimeStoreTest, TestQueryKeyRange) {
     ASSERT_OK(store_->Write(
         RealtimeWriteBatch{MakeBatch(R"([[9, "nine"], [7, "seven"]])"), OffsetRange(2, 4)}));
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store_->AcquireReadView());
-    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(/*include_sequence=*/true);
+    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(FullQueryFields());
     RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
                                  /*enable_predicate_pushdown=*/false};
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
                          store_->CreateQueryReaders(view, /*offset_begin=*/0, context));
     ASSERT_EQ(2, readers.size());
-    auto* first_range = dynamic_cast<PrimaryKeyRangeProvider*>(readers[0].get());
-    auto* second_range = dynamic_cast<PrimaryKeyRangeProvider*>(readers[1].get());
-    ASSERT_NE(nullptr, first_range);
-    ASSERT_NE(nullptr, second_range);
-    ASSERT_EQ(1, first_range->GetMinKey()->GetLong(0));
-    ASSERT_EQ(5, first_range->GetMaxKey()->GetLong(0));
-    ASSERT_EQ(7, second_range->GetMinKey()->GetLong(0));
-    ASSERT_EQ(9, second_range->GetMaxKey()->GetLong(0));
+    const std::vector<std::pair<int64_t, int64_t>> key_ranges = {{1, 5}, {7, 9}};
+    for (size_t i = 0; i < readers.size(); ++i) {
+        auto* range = dynamic_cast<PrimaryKeyRangeProvider*>(readers[i].get());
+        ASSERT_NE(nullptr, range);
+        ASSERT_EQ(key_ranges[i].first, range->GetMinKey()->GetLong(0));
+        ASSERT_EQ(key_ranges[i].second, range->GetMaxKey()->GetLong(0));
+    }
+    AssertReaderOutput(readers, QueryType(),
+                       R"([[0, 6, 1, "one"], [0, 5, 5, "five"], [0, 8, 7, "seven"],
+                            [0, 7, 9, "nine"]])");
+
+    ASSERT_OK(store_->AdvanceCommittedOffset(2));
+    ASSERT_OK_AND_ASSIGN(view, store_->AcquireReadView());
+    read_schema = MakeReadSchema(FullQueryFields());
+    context.read_schema = read_schema.get();
+    ASSERT_OK_AND_ASSIGN(readers, store_->CreateQueryReaders(view, /*offset_begin=*/0, context));
+    ASSERT_EQ(1, readers.size());
+    auto* range = dynamic_cast<PrimaryKeyRangeProvider*>(readers[0].get());
+    ASSERT_NE(nullptr, range);
+    ASSERT_EQ(7, range->GetMinKey()->GetLong(0));
+    ASSERT_EQ(9, range->GetMaxKey()->GetLong(0));
+    AssertReaderOutput(readers, QueryType(), R"([[0, 8, 7, "seven"], [0, 7, 9, "nine"]])");
+}
+
+TEST_F(PrimaryKeyRealtimeStoreTest, TestSequenceExhaustion) {
+    const int64_t max_sequence = std::numeric_limits<int64_t>::max();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                         CreateStore(schema_, {"id"}, max_sequence - 3));
+    ASSERT_OK(store->Write(RealtimeWriteBatch{MakeBatch(R"([[1, "kept"]])"), OffsetRange(10, 11)}));
+    ASSERT_NOK_WITH_MSG(
+        store->Write(RealtimeWriteBatch{
+            MakeBatch(R"([[7, "rejected-a"], [8, "rejected-b"], [9, "rejected-c"]])"),
+            OffsetRange(11, 14)}),
+        "sequence range exceeds INT64_MAX");
+    ASSERT_OK(
+        store->Write(RealtimeWriteBatch{MakeBatch(R"([[2, "also-kept"]])"), OffsetRange(11, 12)}));
+
+    ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
+                         store->SealForCommit());
+    ASSERT_TRUE(segment.has_value());
+    ASSERT_EQ(OffsetRange(10, 12), segment.value()->GetOffsetRange());
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
+    ASSERT_EQ(std::optional<OffsetRange>(OffsetRange(10, 12)), view->GetOffsetRange());
+    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(FullQueryFields());
+    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store->CreateQueryReaders(view, /*offset_begin=*/10, context));
+    AssertReaderOutput(readers, QueryType(),
+                       R"([[0, 9223372036854775805, 1, "kept"],
+                            [0, 9223372036854775806, 2, "also-kept"]])");
+}
+
+TEST_F(PrimaryKeyRealtimeStoreTest, TestQueryProjection) {
+    ASSERT_OK(
+        store_->Write(RealtimeWriteBatch{MakeBatch(R"([[2, "b"], [1, "a"]])"), OffsetRange(0, 2)}));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store_->AcquireReadView());
+    const std::shared_ptr<arrow::Field> value_kind =
+        DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind());
+    const std::shared_ptr<arrow::Field> sequence =
+        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber());
+    struct ProjectionCase {
+        arrow::FieldVector requested;
+        std::shared_ptr<arrow::DataType> expected_type;
+        std::string expected_json;
+    };
+    const std::vector<ProjectionCase> cases = {
+        {{schema_->field(1), value_kind, sequence, schema_->field(0)},
+         arrow::struct_({value_kind, schema_->field(1), sequence, schema_->field(0)}),
+         R"([[0, "a", 6, 1], [0, "b", 5, 2]])"},
+        {{schema_->field(0), value_kind},
+         arrow::struct_({value_kind, schema_->field(0)}),
+         R"([[0, 1], [0, 2]])"},
+    };
+    for (const ProjectionCase& test_case : cases) {
+        std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(test_case.requested);
+        RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
+                                     /*enable_predicate_pushdown=*/false};
+        ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                             store_->CreateQueryReaders(view, /*offset_begin=*/0, context));
+        AssertReaderOutput(readers, test_case.expected_type, test_case.expected_json);
+    }
+
+    std::unique_ptr<ArrowSchema> read_schema =
+        MakeReadSchema({arrow::field("unknown", arrow::int64())});
+    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+    ASSERT_NOK_WITH_MSG(store_->CreateQueryReaders(view, /*offset_begin=*/0, context),
+                        "query field is missing from write schema: unknown");
+}
+
+TEST_F(PrimaryKeyRealtimeStoreTest, TestCompositeKeyClipping) {
+    std::shared_ptr<arrow::Schema> composite_schema =
+        arrow::schema({arrow::field("id", arrow::int64()), arrow::field("region", arrow::utf8()),
+                       arrow::field("value", arrow::utf8())});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                         CreateStore(composite_schema, {"id", "region"},
+                                     /*restore_max_sequence=*/4));
+    ASSERT_OK(store->Write(
+        RealtimeWriteBatch{MakeBatch(R"([[9, "z", "clipped"], [2, "b", "two-b"], [1, "c", "one-c"],
+                       [2, "a", "two-a"]])",
+                                     {}, composite_schema),
+                           OffsetRange(20, 24)}));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
+    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(FullQueryFields(composite_schema));
+    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store->CreateQueryReaders(view, /*offset_begin=*/21, context));
+    ASSERT_EQ(1, readers.size());
+    auto* range = dynamic_cast<PrimaryKeyRangeProvider*>(readers[0].get());
+    ASSERT_NE(nullptr, range);
+    ASSERT_EQ(1, range->GetMinKey()->GetLong(0));
+    ASSERT_EQ("c", range->GetMinKey()->GetString(1).ToString());
+    ASSERT_EQ(2, range->GetMaxKey()->GetLong(0));
+    ASSERT_EQ("b", range->GetMaxKey()->GetString(1).ToString());
+    std::shared_ptr<arrow::DataType> query_type = arrow::struct_(
+        {DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind()),
+         DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber()),
+         composite_schema->field(0), composite_schema->field(1), composite_schema->field(2)});
+    AssertReaderOutput(readers, query_type,
+                       R"([[0, 7, 1, "c", "one-c"], [0, 8, 2, "a", "two-a"],
+                            [0, 6, 2, "b", "two-b"]])");
 }
 
 }  // namespace paimon::test
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index e6000561f..aad9dc2ac 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -646,16 +646,18 @@ class RealtimeWriteInteTest : public ::testing::Test {
                                        /*ignore_if_exists=*/false));
     }
 
-    void CreatePkTable(const std::vector<std::string>& partition_keys = {}) const {
+    void CreatePkTable(const std::vector<std::string>& partition_keys = {},
+                       const std::vector<std::string>& primary_keys = {"id"}) const {
         auto c_schema = std::make_unique<ArrowSchema>();
         ASSERT_TRUE(arrow::ExportSchema(*schema_, c_schema.get()).ok());
         ASSERT_OK_AND_ASSIGN(std::unique_ptr<Catalog> catalog,
                              Catalog::Create(dir_->Str(), options_));
         ASSERT_OK(catalog->CreateDatabase("foo", {}, /*ignore_if_exists=*/false));
-        std::vector<std::string> primary_keys = partition_keys;
-        primary_keys.push_back("id");
+        std::vector<std::string> table_primary_keys = partition_keys;
+        table_primary_keys.insert(table_primary_keys.end(), primary_keys.begin(),
+                                  primary_keys.end());
         ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), c_schema.get(), partition_keys,
-                                       primary_keys, options_, /*ignore_if_exists=*/false));
+                                       table_primary_keys, options_, /*ignore_if_exists=*/false));
     }
 
     Result<std::unique_ptr<FileStoreWrite>> CreateRealtimeWriter(
@@ -692,7 +694,7 @@ class RealtimeWriteInteTest : public ::testing::Test {
         std::string json = "[";
         for (size_t i = 0; i < rows.size(); ++i) {
             const auto& [id, payload, pt] = rows[i];
-            if (pt != partition) {
+            if (partitioned && pt != partition) {
                 return Status::Invalid("one test batch must contain only one partition");
             }
             if (i > 0) {
@@ -1383,6 +1385,242 @@ TEST_F(RealtimeWriteInteTest, TestPkRead) {
     ASSERT_TRUE(query_view->expired());
 }
 
+TEST_F(RealtimeWriteInteTest, TestPkMergeDiskSealedAndActive) {
+    options_[Options::READ_BATCH_SIZE] = "2";
+    CreatePkTable();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+
+    const std::vector<std::vector<Row>> disk_batches = {
+        {{1, "disk-1", "p0"}, {2, "disk-2", "p0"}, {3, "disk-3", "p0"}},
+        {{10, "disk-10", "p0"}, {11, "disk-11", "p0"}},
+    };
+    int64_t commit_identifier = 0;
+    for (const std::vector<Row>& disk_rows : disk_batches) {
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                             MakeBatch(disk_rows, /*partitioned=*/false));
+        ASSERT_OK(writer->Write(std::move(batch)));
+        ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+                             writer->PrepareCommitWithProgress(commit_identifier));
+        ASSERT_EQ(1, progress.size());
+        ASSERT_EQ(1, NewFiles(progress).size());
+        ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, commit_identifier));
+        ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
+        ++commit_identifier;
+    }
+
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<RecordBatch> sealed_batch,
+        MakeBatch({Row{1, "sealed-1", "p0"}, Row{2, "deleted-2", "p0"}, Row{4, "sealed-4", "p0"}},
+                  /*partitioned=*/false, /*bucket=*/0,
+                  {RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::DELETE,
+                   RecordBatch::RowKind::INSERT}));
+    ASSERT_OK(writer->Write(std::move(sealed_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> sealed_progress,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
+    ASSERT_EQ(1, sealed_progress.size());
+
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<RecordBatch> active_batch,
+        MakeBatch({Row{1, "active-1", "p0"}, Row{4, "deleted-4", "p0"}, Row{5, "active-5", "p0"}},
+                  /*partitioned=*/false, /*bucket=*/0,
+                  {RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::DELETE,
+                   RecordBatch::RowKind::INSERT}));
+    ASSERT_OK(writer->Write(std::move(active_batch)));
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
+                         CreatePlan(realtime_context, /*predicate=*/nullptr));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult result,
+                         ReadPlan(plan, realtime_context, {"payload", "id"}, /*predicate=*/nullptr,
+                                  /*enable_predicate_filter=*/false));
+    ASSERT_NE(nullptr, result.data);
+    ASSERT_GT(result.data->num_chunks(), 1);
+    for (const std::shared_ptr<arrow::Array>& chunk : result.data->chunks()) {
+        ASSERT_LE(chunk->length(), 2);
+    }
+    std::shared_ptr<arrow::DataType> result_type = arrow::struct_(
+        {arrow::field("_VALUE_KIND", arrow::int8()), arrow::field("payload", arrow::utf8()),
+         arrow::field("id", arrow::int64())});
+    std::shared_ptr<arrow::Array> expected =
+        arrow::ipc::internal::json::ArrayFromJSON(result_type, R"([
+            [0, "active-1", 1],
+            [0, "disk-3", 3],
+            [0, "active-5", 5],
+            [0, "disk-10", 10],
+            [0, "disk-11", 11]
+        ])")
+            .ValueOrDie();
+    ASSERT_TRUE(std::make_shared<arrow::ChunkedArray>(expected)->Equals(*result.data))
+        << result.data->ToString();
+    result.reader->Close();
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestPkCompositeMerge) {
+    CreatePkTable(/*partition_keys=*/{}, /*primary_keys=*/{"id", "payload"});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
+                         MakeBatch({Row{1, "a", "disk-1a"}, Row{1, "b", "disk-1b"},
+                                    Row{2, "a", "disk-2a"}, Row{3, "c", "disk-3c"}},
+                                   /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(disk_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_progress,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_EQ(1, disk_progress.size());
+    ASSERT_EQ(OffsetRange(0, 4), disk_progress[0].offset_range);
+    ASSERT_EQ(1, NewFiles(disk_progress).size());
+    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(disk_progress, /*commit_identifier=*/0));
+    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
+
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<RecordBatch> sealed_batch,
+        MakeBatch({Row{1, "a", "sealed-1a"}, Row{1, "b", "deleted-1b"}, Row{2, "b", "sealed-2b"}},
+                  /*partitioned=*/false, /*bucket=*/0,
+                  {RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::DELETE,
+                   RecordBatch::RowKind::INSERT}));
+    ASSERT_OK(writer->Write(std::move(sealed_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> sealed_progress,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_EQ(1, sealed_progress.size());
+    ASSERT_EQ(OffsetRange(4, 7), sealed_progress[0].offset_range);
+    ASSERT_EQ(1, NewFiles(sealed_progress).size());
+
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<RecordBatch> active_batch,
+        MakeBatch({Row{1, "a", "active-1a"}, Row{1, "c", "active-1c"}, Row{2, "a", "active-2a"}},
+                  /*partitioned=*/false, /*bucket=*/0,
+                  {RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::INSERT,
+                   RecordBatch::RowKind::UPDATE_AFTER}));
+    ASSERT_OK(writer->Write(std::move(active_batch)));
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
+                         CreatePlan(realtime_context, /*predicate=*/nullptr));
+    ASSERT_EQ(1, plan->Splits().size());
+    std::shared_ptr<RealtimeSplit> split =
+        std::dynamic_pointer_cast<RealtimeSplit>(plan->Splits()[0]);
+    ASSERT_NE(nullptr, split);
+    ASSERT_FALSE(split->DiskSplits().empty());
+    ASSERT_EQ(4, split->CommittedEndOffset());
+    ASSERT_EQ(10, split->MemoryEndOffset());
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(plan, realtime_context));
+    ASSERT_EQ((std::vector<Row>{{1, "a", "active-1a"},
+                                {1, "c", "active-1c"},
+                                {2, "a", "active-2a"},
+                                {2, "b", "sealed-2b"},
+                                {3, "c", "disk-3c"}}),
+              actual_rows);
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestPkPartitionBucketRecovery) {
+    options_[Options::BUCKET] = "2";
+    CreatePkTable(/*partition_keys=*/{"pt"});
+    const RealtimePartitionBucket p0b0({{"pt", "p0"}}, /*bucket=*/0);
+    const RealtimePartitionBucket p1b1({{"pt", "p1"}}, /*bucket=*/1);
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> first_context, RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> first_writer,
+                         CreateRealtimeWriter(first_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p0_first_batch,
+                         MakeBatch({Row{0, "p0-zero", "p0"}, Row{1, "p0-one", "p0"}},
+                                   /*partitioned=*/true, /*bucket=*/0));
+    ASSERT_OK(first_writer->Write(std::move(p0_first_batch)));
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<RecordBatch> p1_first_batch,
+        MakeBatch({Row{10, "p1-ten", "p1"}, Row{11, "p1-eleven", "p1"}, Row{12, "p1-twelve", "p1"}},
+                  /*partitioned=*/true, /*bucket=*/1));
+    ASSERT_OK(first_writer->Write(std::move(p1_first_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> first_progress,
+                         first_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_EQ(2, first_progress.size());
+    std::map<RealtimePartitionBucket, OffsetRange> first_ranges;
+    std::map<RealtimePartitionBucket, std::pair<int64_t, int64_t>> first_sequences;
+    for (const RealtimeCommitProgress& progress : first_progress) {
+        first_ranges.emplace(progress.partition_bucket, progress.offset_range);
+        std::shared_ptr<CommitMessageImpl> message =
+            std::dynamic_pointer_cast<CommitMessageImpl>(progress.commit_message);
+        ASSERT_NE(nullptr, message);
+        const std::vector<std::shared_ptr<DataFileMeta>>& files =
+            message->GetNewFilesIncrement().NewFiles();
+        ASSERT_EQ(1, files.size());
+        first_sequences.emplace(
+            progress.partition_bucket,
+            std::make_pair(files[0]->min_sequence_number, files[0]->max_sequence_number));
+    }
+    ASSERT_EQ(OffsetRange(0, 2), first_ranges.at(p0b0));
+    ASSERT_EQ(OffsetRange(0, 3), first_ranges.at(p1b1));
+    ASSERT_EQ((std::make_pair<int64_t, int64_t>(0, 1)), first_sequences.at(p0b0));
+    ASSERT_EQ((std::make_pair<int64_t, int64_t>(0, 2)), first_sequences.at(p1b1));
+    ASSERT_OK_AND_ASSIGN(int64_t first_snapshot_id,
+                         Commit(first_progress, /*commit_identifier=*/0));
+    ASSERT_OK(first_writer->RefreshCommittedSnapshot(first_snapshot_id));
+    ASSERT_OK(first_writer->Close());
+    first_writer.reset();
+    first_context.reset();
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> second_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> second_writer,
+                         CreateRealtimeWriter(second_context));
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<RecordBatch> p0_second_batch,
+        MakeBatch({Row{0, "p0-zero-new", "p0"}, Row{2, "p0-two", "p0"}},
+                  /*partitioned=*/true, /*bucket=*/0,
+                  {RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::INSERT}));
+    ASSERT_OK(second_writer->Write(std::move(p0_second_batch)));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p1_second_batch,
+                         MakeBatch({Row{10, "p1-ten-deleted", "p1"}, Row{13, "p1-thirteen", "p1"}},
+                                   /*partitioned=*/true, /*bucket=*/1,
+                                   {RecordBatch::RowKind::DELETE, RecordBatch::RowKind::INSERT}));
+    ASSERT_OK(second_writer->Write(std::move(p1_second_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_progress,
+                         second_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_EQ(2, second_progress.size());
+    std::map<RealtimePartitionBucket, OffsetRange> second_ranges;
+    std::map<RealtimePartitionBucket, std::pair<int64_t, int64_t>> second_sequences;
+    for (const RealtimeCommitProgress& progress : second_progress) {
+        second_ranges.emplace(progress.partition_bucket, progress.offset_range);
+        std::shared_ptr<CommitMessageImpl> message =
+            std::dynamic_pointer_cast<CommitMessageImpl>(progress.commit_message);
+        ASSERT_NE(nullptr, message);
+        const std::vector<std::shared_ptr<DataFileMeta>>& files =
+            message->GetNewFilesIncrement().NewFiles();
+        ASSERT_EQ(1, files.size());
+        second_sequences.emplace(
+            progress.partition_bucket,
+            std::make_pair(files[0]->min_sequence_number, files[0]->max_sequence_number));
+    }
+    ASSERT_EQ(OffsetRange(2, 4), second_ranges.at(p0b0));
+    ASSERT_EQ(OffsetRange(3, 5), second_ranges.at(p1b1));
+    ASSERT_EQ((std::make_pair<int64_t, int64_t>(2, 3)), second_sequences.at(p0b0));
+    ASSERT_EQ((std::make_pair<int64_t, int64_t>(3, 4)), second_sequences.at(p1b1));
+    ASSERT_OK_AND_ASSIGN(int64_t second_snapshot_id,
+                         Commit(second_progress, /*commit_identifier=*/1));
+    ASSERT_OK(second_writer->RefreshCommittedSnapshot(second_snapshot_id));
+
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(second_context));
+    std::sort(actual_rows.begin(), actual_rows.end());
+    ASSERT_EQ((std::vector<Row>{{0, "p0-zero-new", "p0"},
+                                {1, "p0-one", "p0"},
+                                {2, "p0-two", "p0"},
+                                {11, "p1-eleven", "p1"},
+                                {12, "p1-twelve", "p1"},
+                                {13, "p1-thirteen", "p1"}}),
+              actual_rows);
+    ASSERT_OK(second_writer->Close());
+
+    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap offsets, ReadCommittedOffsets());
+    ASSERT_EQ(2, offsets.size());
+    ASSERT_EQ(4, offsets.at(p0b0));
+    ASSERT_EQ(5, offsets.at(p1b1));
+}
+
 TEST_F(RealtimeWriteInteTest, TestPkRecovery) {
     CreatePkTable();
 

From b3862d119eb26004a769808ce3f2cb8e37e97c87 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Fri, 21 Aug 2026 12:38:57 +0800
Subject: [PATCH 10/62] fix(realtime): prevent sequence reuse and align nested
 projections

---
 .../realtime/primary_key_realtime_store.cpp   |  9 +++
 .../primary_key_realtime_store_test.cpp       | 28 +++++++
 .../core/realtime/realtime_context_impl.cpp   | 10 ++-
 .../core/realtime/realtime_context_test.cpp   | 16 +++-
 test/inte/realtime_write_inte_test.cpp        | 81 +++++++++++++++++++
 5 files changed, 139 insertions(+), 5 deletions(-)

diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index afdc0c73c..7999de75d 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -43,6 +43,7 @@
 #include "paimon/core/io/merged_key_value_record_reader.h"
 #include "paimon/core/key_value.h"
 #include "paimon/core/mergetree/compact/sort_merge_reader_with_loser_tree.h"
+#include "paimon/core/utils/nested_projection_utils.h"
 #include "paimon/macros.h"
 
 namespace paimon {
@@ -411,6 +412,7 @@ class PrimaryKeyRealtimeStore::Impl {
                                           arrow::ImportSchema(context.read_schema));
         arrow::FieldVector output_fields = {
             DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())};
+        arrow::FieldVector aligned_value_fields = write_schema_->fields();
         std::vector<int32_t> projection = {KeyValueProjectionConsumer::kValueKindProjection};
         for (const std::shared_ptr<arrow::Field>& field : requested->fields()) {
             if (field->name() == SpecialFields::ValueKind().Name()) {
@@ -426,8 +428,11 @@ class PrimaryKeyRealtimeStore::Impl {
                 return Status::Invalid("PK real-time query field is missing from write schema: ",
                                        field->name());
             }
+            aligned_value_fields[index] = field;
             projection.push_back(index);
         }
+        const std::shared_ptr<arrow::DataType> aligned_value_type =
+            arrow::struct_(aligned_value_fields);
 
         std::vector<std::unique_ptr<BatchReader>> result;
         for (const BatchGroup& group : typed->Groups()) {
@@ -452,6 +457,10 @@ class PrimaryKeyRealtimeStore::Impl {
                 if (!max_key || key_comparator_->CompareTo(*key_range.second, *max_key) > 0) {
                     max_key = key_range.second;
                 }
+                PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> aligned,
+                                       NestedProjectionUtils::AlignArrayToReadType(
+                                           selected, aligned_value_type, arrow_pool_.get()));
+                selected = checked_pointer_cast<arrow::StructArray>(aligned);
                 std::vector<RecordBatch::RowKind> selected_kinds;
                 if (!batch->row_kinds.empty()) {
                     selected_kinds.assign(batch->row_kinds.begin() + offset,
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index 5c04d4310..ef293e54b 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -392,6 +392,34 @@ TEST_F(PrimaryKeyRealtimeStoreTest, TestQueryProjection) {
                         "query field is missing from write schema: unknown");
 }
 
+TEST_F(PrimaryKeyRealtimeStoreTest, TestNestedProjection) {
+    const std::shared_ptr<arrow::Field> id =
+        DataField::ConvertDataFieldToArrowField(DataField(0, arrow::field("id", arrow::int64())));
+    const std::shared_ptr<arrow::Field> a =
+        DataField::ConvertDataFieldToArrowField(DataField(10, arrow::field("a", arrow::int64())));
+    const std::shared_ptr<arrow::Field> b =
+        DataField::ConvertDataFieldToArrowField(DataField(11, arrow::field("b", arrow::int64())));
+    const std::shared_ptr<arrow::Field> payload = DataField::ConvertDataFieldToArrowField(
+        DataField(1, arrow::field("payload", arrow::struct_({a, b}))));
+    const std::shared_ptr<arrow::Schema> nested_schema = arrow::schema({id, payload});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                         CreateStore(nested_schema, {"id"}, /*restore_max_sequence=*/4));
+    ASSERT_OK(store->Write(RealtimeWriteBatch{
+        MakeBatch(R"([[2, [200, 2000]], [1, [100, null]], [3, [300, 3000]]])", {}, nested_schema),
+        OffsetRange(0, 3)}));
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
+    const std::shared_ptr<arrow::Field> projected_payload = payload->WithType(arrow::struct_({b}));
+    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema({projected_payload});
+    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store->CreateQueryReaders(view, /*offset_begin=*/0, context));
+    const std::shared_ptr<arrow::DataType> result_type = arrow::struct_(
+        {DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind()), projected_payload});
+    AssertReaderOutput(readers, result_type, R"([[0, [null]], [0, [2000]], [0, [3000]]])");
+}
+
 TEST_F(PrimaryKeyRealtimeStoreTest, TestCompositeKeyClipping) {
     std::shared_ptr<arrow::Schema> composite_schema =
         arrow::schema({arrow::field("id", arrow::int64()), arrow::field("region", arrow::utf8()),
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index 6624059a6..066e54e8a 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -82,6 +82,7 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     std::lock_guard<std::mutex> progress_lock(progress_mutex_);
     std::lock_guard<std::mutex> registry_lock(mutex_);
     const RealtimePartitionBucket key(request.partition, request.bucket);
+    auto iter = stores_.find(key);
     std::optional<int64_t> initial_max_sequence_number;
     PrimaryKeyRealtimeStoreCreateConfig* primary_key_config =
         std::get_if<PrimaryKeyRealtimeStoreCreateConfig>(&request.mode_config);
@@ -89,6 +90,14 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
         auto [sequence_iter, inserted] = materialized_max_sequence_numbers_.emplace(
             key, primary_key_config->restore_max_sequence_number);
         if (!inserted && primary_key_config->restore_max_sequence_number > sequence_iter->second) {
+            if (iter != stores_.end()) {
+                if (request.write_schema) {
+                    ArrowSchemaRelease(request.write_schema.get());
+                }
+                return Status::Invalid(
+                    "restore max sequence number exceeds the materialized watermark of an "
+                    "existing PK real-time store");
+            }
             sequence_iter->second = primary_key_config->restore_max_sequence_number;
         }
         initial_max_sequence_number = sequence_iter->second;
@@ -105,7 +114,6 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
         }
         initial_offset = offset_iter->second;
     }
-    auto iter = stores_.find(key);
     if (iter != stores_.end()) {
         if (request.write_schema) {
             ArrowSchemaRelease(request.write_schema.get());
diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp
index b4d2c6718..ab0abe4a7 100644
--- a/src/paimon/core/realtime/realtime_context_test.cpp
+++ b/src/paimon/core/realtime/realtime_context_test.cpp
@@ -198,12 +198,20 @@ TEST(RealtimeContextTest, TestReconcilesPrimaryKeyInitialSequence) {
     ASSERT_EQ(first_state.store, retained_state.store);
     ASSERT_EQ(8, retained_state.initial_max_sequence_number);
 
-    ASSERT_OK_AND_ASSIGN(
-        RealtimeStoreState restored_state,
+    ASSERT_NOK_WITH_MSG(
         GetOrCreatePrimaryKeyStore(context, partition, /*bucket=*/0,
+                                   /*restore_max_sequence_number=*/10, GetDefaultPool()),
+        "restore max sequence number exceeds the materialized watermark of an "
+        "existing PK real-time store");
+
+    const RealtimePartitionBucket new_partition_bucket(partition, /*bucket=*/1);
+    context->AdvanceMaterializedMaxSequenceNumber(new_partition_bucket,
+                                                  /*max_sequence_number=*/8);
+    ASSERT_OK_AND_ASSIGN(
+        RealtimeStoreState new_state,
+        GetOrCreatePrimaryKeyStore(context, partition, /*bucket=*/1,
                                    /*restore_max_sequence_number=*/10, GetDefaultPool()));
-    ASSERT_EQ(first_state.store, restored_state.store);
-    ASSERT_EQ(10, restored_state.initial_max_sequence_number);
+    ASSERT_EQ(10, new_state.initial_max_sequence_number);
 }
 
 TEST(RealtimeContextTest, TestCommittedProgressIsMonotonicAndSelective) {
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index aad9dc2ac..9f302eb37 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -1458,6 +1458,87 @@ TEST_F(RealtimeWriteInteTest, TestPkMergeDiskSealedAndActive) {
     ASSERT_OK(writer->Close());
 }
 
+TEST_F(RealtimeWriteInteTest, TestPkNestedProjectionAcrossDiskAndMemory) {
+    const std::shared_ptr<arrow::Field> projected_b = arrow::field("b", arrow::int64());
+    fields_ = {
+        arrow::field("id", arrow::int64()),
+        arrow::field("payload", arrow::struct_({arrow::field("a", arrow::int64()), projected_b})),
+        arrow::field("pt", arrow::utf8()),
+    };
+    schema_ = arrow::schema(fields_);
+    CreatePkTable();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+    auto make_batch = [&](const std::string& json) -> Result<std::unique_ptr<RecordBatch>> {
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+            std::shared_ptr<arrow::Array> array,
+            arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_), json));
+        ArrowArray c_array;
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array));
+        RecordBatchBuilder builder(&c_array);
+        return builder.SetBucket(0).Finish();
+    };
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
+                         make_batch(R"([[1, [101, 1001], "p0"], [2, [102, 1002], "p0"]])"));
+    ASSERT_OK(writer->Write(std::move(disk_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_progress,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(disk_progress, /*commit_identifier=*/0));
+    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> sealed_batch,
+                         make_batch(R"([[1, [201, 2001], "p0"], [3, [203, 2003], "p0"]])"));
+    ASSERT_OK(writer->Write(std::move(sealed_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> sealed_progress,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_EQ(1, sealed_progress.size());
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> active_batch,
+                         make_batch(R"([[1, [301, 3001], "p0"], [4, [304, null], "p0"]])"));
+    ASSERT_OK(writer->Write(std::move(active_batch)));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
+                         CreatePlan(realtime_context, /*predicate=*/nullptr));
+
+    auto projected_schema = arrow::schema({
+        arrow::field("payload", arrow::struct_({projected_b})),
+        arrow::field("id", arrow::int64()),
+    });
+    auto c_schema = std::make_unique<ArrowSchema>();
+    ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok());
+    ReadContextBuilder read_builder(table_path_);
+    read_builder.SetOptions(options_)
+        .SetReadSchema(std::move(c_schema))
+        .WithRealtimeContext(realtime_context)
+        .WithMemoryPool(pool_);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableRead> table_read,
+                         TableRead::Create(std::move(read_context)));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> reader,
+                         table_read->CreateReader(plan->Splits()));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> actual,
+                         ReadResultCollector::CollectResult(reader.get()));
+    const std::shared_ptr<arrow::DataType> result_type = arrow::struct_({
+        arrow::field("_VALUE_KIND", arrow::int8()),
+        arrow::field("payload", arrow::struct_({projected_b})),
+        arrow::field("id", arrow::int64()),
+    });
+    const std::shared_ptr<arrow::Array> expected =
+        arrow::ipc::internal::json::ArrayFromJSON(result_type, R"([
+            [0, [3001], 1],
+            [0, [1002], 2],
+            [0, [2003], 3],
+            [0, [null], 4]
+        ])")
+            .ValueOrDie();
+    ASSERT_TRUE(std::make_shared<arrow::ChunkedArray>(expected)->Equals(*actual))
+        << actual->ToString();
+    reader->Close();
+    ASSERT_OK(writer->Close());
+}
+
 TEST_F(RealtimeWriteInteTest, TestPkCompositeMerge) {
     CreatePkTable(/*partition_keys=*/{}, /*primary_keys=*/{"id", "payload"});
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,

From 3acae038e0446cfb4d92f8572e51b13d3e00b63e Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Fri, 21 Aug 2026 13:20:03 +0800
Subject: [PATCH 11/62] fix(realtime): align PK reads across schema changes

---
 .../realtime/primary_key_realtime_store.cpp   |   28 +-
 test/inte/CMakeLists.txt                      |    7 +
 ...chema_evolution_write_verify_inte_test.cpp | 1110 +++++++++++++++++
 3 files changed, 1136 insertions(+), 9 deletions(-)
 create mode 100644 test/inte/schema_evolution_write_verify_inte_test.cpp

diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index 7999de75d..6565ed8d7 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -423,12 +423,18 @@ class PrimaryKeyRealtimeStore::Impl {
                 projection.push_back(KeyValueProjectionConsumer::kSequenceNumberProjection);
                 continue;
             }
-            const int32_t index = write_schema_->GetFieldIndex(field->name());
+            int32_t index = write_schema_->GetFieldIndex(field->name());
             if (index < 0) {
-                return Status::Invalid("PK real-time query field is missing from write schema: ",
-                                       field->name());
+                Result<int32_t> field_id = NestedProjectionUtils::GetPaimonFieldId(field);
+                if (!field_id.ok()) {
+                    return Status::Invalid(
+                        "PK real-time query field is missing from write schema: ", field->name());
+                }
+                index = static_cast<int32_t>(aligned_value_fields.size());
+                aligned_value_fields.push_back(field);
+            } else {
+                aligned_value_fields[index] = field;
             }
-            aligned_value_fields[index] = field;
             projection.push_back(index);
         }
         const std::shared_ptr<arrow::DataType> aligned_value_type =
@@ -446,8 +452,16 @@ class PrimaryKeyRealtimeStore::Impl {
                 const int64_t offset = std::max<int64_t>(0, lower - batch->offset_range.begin);
                 const int64_t length = batch->data->length() - offset;
                 std::shared_ptr<arrow::Array> sliced = batch->data->Slice(offset, length);
+                PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> aligned,
+                                       NestedProjectionUtils::AlignArrayToReadType(
+                                           sliced, aligned_value_type, arrow_pool_.get()));
+                if (!aligned || aligned->type_id() != arrow::Type::STRUCT) {
+                    return Status::Invalid(
+                        "PK real-time query projection did not produce a "
+                        "StructArray");
+                }
                 std::shared_ptr<arrow::StructArray> selected =
-                    checked_pointer_cast<arrow::StructArray>(sliced);
+                    checked_pointer_cast<arrow::StructArray>(aligned);
                 using KeyRange =
                     std::pair<std::shared_ptr<InternalRow>, std::shared_ptr<InternalRow>>;
                 PAIMON_ASSIGN_OR_RAISE(KeyRange key_range, GetKeyRange(selected));
@@ -457,10 +471,6 @@ class PrimaryKeyRealtimeStore::Impl {
                 if (!max_key || key_comparator_->CompareTo(*key_range.second, *max_key) > 0) {
                     max_key = key_range.second;
                 }
-                PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> aligned,
-                                       NestedProjectionUtils::AlignArrayToReadType(
-                                           selected, aligned_value_type, arrow_pool_.get()));
-                selected = checked_pointer_cast<arrow::StructArray>(aligned);
                 std::vector<RecordBatch::RowKind> selected_kinds;
                 if (!batch->row_kinds.empty()) {
                     selected_kinds.assign(batch->row_kinds.begin() + offset,
diff --git a/test/inte/CMakeLists.txt b/test/inte/CMakeLists.txt
index 75147ce60..f1b3f8ce6 100644
--- a/test/inte/CMakeLists.txt
+++ b/test/inte/CMakeLists.txt
@@ -43,6 +43,13 @@ if(PAIMON_BUILD_TESTS)
                     test_utils_static
                     ${GTEST_LINK_TOOLCHAIN})
 
+    add_paimon_test(schema_evolution_write_verify_inte_test
+                    STATIC_LINK_LIBS
+                    paimon_shared
+                    ${TEST_STATIC_LINK_LIBS}
+                    test_utils_static
+                    ${GTEST_LINK_TOOLCHAIN})
+
     add_paimon_test(global_index_test
                     STATIC_LINK_LIBS
                     paimon_shared
diff --git a/test/inte/schema_evolution_write_verify_inte_test.cpp b/test/inte/schema_evolution_write_verify_inte_test.cpp
new file mode 100644
index 000000000..dcadbd9e1
--- /dev/null
+++ b/test/inte/schema_evolution_write_verify_inte_test.cpp
@@ -0,0 +1,1110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstddef>
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "arrow/api.h"
+#include "arrow/c/bridge.h"
+#include "arrow/ipc/json_simple.h"
+#include "gtest/gtest.h"
+#include "paimon/catalog/catalog.h"
+#include "paimon/catalog/identifier.h"
+#include "paimon/commit_context.h"
+#include "paimon/common/utils/path_util.h"
+#include "paimon/common/utils/scope_guard.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/global_index/indexed_split_impl.h"
+#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/schema/schema_manager.h"
+#include "paimon/core/snapshot.h"
+#include "paimon/core/table/sink/commit_message_impl.h"
+#include "paimon/core/table/source/data_split_impl.h"
+#include "paimon/defs.h"
+#include "paimon/file_index/file_index_format.h"
+#include "paimon/file_index/file_index_reader.h"
+#include "paimon/file_index/file_index_result.h"
+#include "paimon/file_store_commit.h"
+#include "paimon/file_store_write.h"
+#include "paimon/fs/file_system.h"
+#include "paimon/io/byte_array_input_stream.h"
+#include "paimon/predicate/literal.h"
+#include "paimon/predicate/predicate_builder.h"
+#include "paimon/read_context.h"
+#include "paimon/reader/batch_reader.h"
+#include "paimon/realtime/realtime_context.h"
+#include "paimon/record_batch.h"
+#include "paimon/scan_context.h"
+#include "paimon/table/source/plan.h"
+#include "paimon/table/source/startup_mode.h"
+#include "paimon/table/source/table_read.h"
+#include "paimon/table/source/table_scan.h"
+#include "paimon/testing/utils/read_result_collector.h"
+#include "paimon/testing/utils/test_helper.h"
+#include "paimon/testing/utils/testharness.h"
+#include "paimon/write_context.h"
+
+namespace paimon::test {
+namespace {
+
+std::map<std::string, std::string> BaseOptions() {
+    return {{Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, "orc"},
+            {Options::FILE_SYSTEM, "local"},   {Options::BUCKET, "1"},
+            {Options::BUCKET_KEY, "id"},       {Options::TARGET_FILE_SIZE, "1MB"}};
+}
+
+std::map<std::string, std::string> DataEvolutionOptions() {
+    return {{Options::MANIFEST_FORMAT, "orc"},       {Options::FILE_FORMAT, "orc"},
+            {Options::FILE_SYSTEM, "local"},         {Options::TARGET_FILE_SIZE, "1MB"},
+            {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}};
+}
+
+arrow::FieldVector BaseFields() {
+    return {arrow::field("id", arrow::int64()), arrow::field("payload", arrow::utf8())};
+}
+
+arrow::FieldVector EvolvedFields() {
+    return {arrow::field("id", arrow::int64()), arrow::field("payload", arrow::utf8()),
+            arrow::field("extra", arrow::int32())};
+}
+
+arrow::FieldVector DataEvolutionFields() {
+    return {arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::utf8()),
+            arrow::field("f2", arrow::utf8())};
+}
+
+Result<std::unique_ptr<RecordBatch>> MakeBatch(
+    const arrow::FieldVector& fields, const std::string& json,
+    const std::map<std::string, std::string>& partition, int32_t bucket,
+    const std::vector<RecordBatch::RowKind>& row_kinds = {}) {
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+        std::shared_ptr<arrow::Array> array,
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), json));
+    ArrowArray c_array;
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array));
+    RecordBatchBuilder builder(&c_array);
+    return builder.SetPartition(partition).SetBucket(bucket).SetRowKinds(row_kinds).Finish();
+}
+
+Result<std::unique_ptr<RecordBatch>> MakeUnbucketedBatch(
+    const arrow::FieldVector& fields, const std::string& json,
+    const std::map<std::string, std::string>& partition,
+    const std::vector<RecordBatch::RowKind>& row_kinds = {}) {
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+        std::shared_ptr<arrow::Array> array,
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), json));
+    ArrowArray c_array;
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array));
+    RecordBatchBuilder builder(&c_array);
+    return builder.SetPartition(partition).SetRowKinds(row_kinds).Finish();
+}
+
+Result<std::unique_ptr<FileStoreWrite>> CreateWriter(
+    const std::string& table_path, const std::map<std::string, std::string>& options,
+    const std::shared_ptr<RealtimeContext>& realtime_context = nullptr,
+    const std::vector<std::string>& write_schema = {}) {
+    WriteContextBuilder builder(table_path, "schema_evolution_verify");
+    builder.SetOptions(options).WithStreamingMode(true);
+    if (realtime_context) {
+        builder.WithRealtimeContext(realtime_context);
+    }
+    if (!write_schema.empty()) {
+        builder.WithWriteSchema(write_schema);
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<WriteContext> context, builder.Finish());
+    return FileStoreWrite::Create(std::move(context));
+}
+
+Result<std::vector<std::shared_ptr<CommitMessage>>> WriteWithNewWriter(
+    const std::string& table_path, const std::map<std::string, std::string>& options,
+    std::unique_ptr<RecordBatch> batch, int64_t commit_identifier,
+    const std::vector<std::string>& write_schema = {}) {
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreWrite> writer,
+                           CreateWriter(table_path, options, nullptr, write_schema));
+    PAIMON_RETURN_NOT_OK(writer->Write(std::move(batch)));
+    PAIMON_ASSIGN_OR_RAISE(std::vector<std::shared_ptr<CommitMessage>> messages,
+                           writer->PrepareCommit(/*wait_compaction=*/false, commit_identifier));
+    PAIMON_RETURN_NOT_OK(writer->Close());
+    return messages;
+}
+
+Result<std::unique_ptr<FileStoreCommit>> CreateCommit(
+    const std::string& table_path, const std::map<std::string, std::string>& options) {
+    CommitContextBuilder builder(table_path, "schema_evolution_verify");
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<CommitContext> context,
+                           builder.SetOptions(options).IgnoreEmptyCommit(false).Finish());
+    return FileStoreCommit::Create(std::move(context));
+}
+
+Status CommitMessages(const std::string& table_path,
+                      const std::map<std::string, std::string>& options,
+                      const std::vector<std::shared_ptr<CommitMessage>>& messages,
+                      int64_t commit_identifier) {
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreCommit> commit,
+                           CreateCommit(table_path, options));
+    return commit->Commit(messages, commit_identifier);
+}
+
+Result<int64_t> CommitRealtimeMessages(const std::string& table_path,
+                                       const std::map<std::string, std::string>& options,
+                                       const std::vector<RealtimeCommitProgress>& messages,
+                                       int64_t commit_identifier) {
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreCommit> commit,
+                           CreateCommit(table_path, options));
+    return commit->CommitWithProgress(messages, commit_identifier, /*watermark=*/std::nullopt);
+}
+
+Result<std::optional<Snapshot>> LatestSnapshot(const std::string& table_path,
+                                               const std::map<std::string, std::string>& options,
+                                               const std::shared_ptr<FileSystem>& file_system) {
+    PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options, file_system));
+    SnapshotManager snapshot_manager(core_options.GetFileSystem(), table_path);
+    return snapshot_manager.LatestSnapshot();
+}
+
+Result<std::shared_ptr<Plan>> ScanTable(
+    const std::string& table_path, const std::map<std::string, std::string>& options,
+    const std::shared_ptr<MemoryPool>& pool,
+    const std::shared_ptr<RealtimeContext>& realtime_context = nullptr,
+    const std::shared_ptr<Predicate>& predicate = nullptr) {
+    ScanContextBuilder scan_builder(table_path);
+    scan_builder.SetOptions(options)
+        .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString())
+        .SetPredicate(predicate)
+        .WithMemoryPool(pool);
+    if (realtime_context) {
+        scan_builder.WithRealtimeContext(realtime_context);
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ScanContext> scan_context, scan_builder.Finish());
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableScan> table_scan,
+                           TableScan::Create(std::move(scan_context)));
+    return table_scan->CreatePlan();
+}
+
+std::vector<std::shared_ptr<DataFileMeta>> DataFilesFromPlan(const std::shared_ptr<Plan>& plan) {
+    std::vector<std::shared_ptr<DataFileMeta>> files;
+    for (const std::shared_ptr<Split>& split : plan->Splits()) {
+        std::shared_ptr<Split> data_split = split;
+        if (std::shared_ptr<IndexedSplitImpl> indexed_split =
+                std::dynamic_pointer_cast<IndexedSplitImpl>(split)) {
+            data_split = indexed_split->GetDataSplit();
+        }
+        std::shared_ptr<DataSplitImpl> split_impl =
+            std::dynamic_pointer_cast<DataSplitImpl>(data_split);
+        if (!split_impl) {
+            continue;
+        }
+        const std::vector<std::shared_ptr<DataFileMeta>>& split_files = split_impl->DataFiles();
+        files.insert(files.end(), split_files.begin(), split_files.end());
+    }
+    return files;
+}
+
+size_t CountIndexedSplits(const std::shared_ptr<Plan>& plan) {
+    size_t count = 0;
+    for (const std::shared_ptr<Split>& split : plan->Splits()) {
+        if (std::dynamic_pointer_cast<IndexedSplitImpl>(split)) {
+            count++;
+        }
+    }
+    return count;
+}
+
+Status EvolveSchema(const std::string& table_path, const std::shared_ptr<FileSystem>& file_system,
+                    const std::vector<DataField>& fields, int32_t highest_field_id,
+                    const std::map<std::string, std::string>& options) {
+    return TestHelper::WriteNextSchema(file_system, table_path, fields, highest_field_id, options);
+}
+
+void AssignFirstRowId(const std::vector<std::shared_ptr<CommitMessage>>& messages,
+                      int64_t first_row_id) {
+    for (const std::shared_ptr<CommitMessage>& commit_message : messages) {
+        std::shared_ptr<CommitMessageImpl> message =
+            std::dynamic_pointer_cast<CommitMessageImpl>(commit_message);
+        ASSERT_TRUE(message);
+        for (const std::shared_ptr<DataFileMeta>& file :
+             message->GetNewFilesIncrement().NewFiles()) {
+            file->AssignFirstRowId(first_row_id);
+        }
+    }
+}
+
+struct CollectedReadResult {
+    std::unique_ptr<TableRead> table_read;
+    std::unique_ptr<BatchReader> reader;
+    std::shared_ptr<arrow::ChunkedArray> data;
+};
+
+Result<CollectedReadResult> ReadRows(
+    const std::string& table_path, const std::map<std::string, std::string>& options,
+    const std::shared_ptr<MemoryPool>& pool,
+    const std::shared_ptr<RealtimeContext>& realtime_context = nullptr,
+    const std::shared_ptr<Predicate>& predicate = nullptr, bool enable_predicate_filter = true) {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Plan> plan,
+                           ScanTable(table_path, options, pool, realtime_context, predicate));
+
+    ReadContextBuilder read_builder(table_path);
+    read_builder.SetOptions(options)
+        .SetPredicate(predicate)
+        .EnablePredicateFilter(enable_predicate_filter)
+        .WithMemoryPool(pool);
+    if (realtime_context) {
+        read_builder.WithRealtimeContext(realtime_context);
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableRead> table_read,
+                           TableRead::Create(std::move(read_context)));
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> batch_reader,
+                           table_read->CreateReader(plan->Splits()));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::ChunkedArray> rows,
+                           ReadResultCollector::CollectResult(batch_reader.get()));
+    return CollectedReadResult{std::move(table_read), std::move(batch_reader), std::move(rows)};
+}
+
+void AssertResultEquals(const std::shared_ptr<arrow::ChunkedArray>& actual,
+                        const arrow::FieldVector& fields, const std::string& expected_json) {
+    arrow::FieldVector fields_with_row_kind = fields;
+    fields_with_row_kind.insert(fields_with_row_kind.begin(),
+                                arrow::field("_VALUE_KIND", arrow::int8()));
+    std::shared_ptr<arrow::Array> expected_array =
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_with_row_kind),
+                                                  expected_json)
+            .ValueOrDie();
+    auto expected = std::make_shared<arrow::ChunkedArray>(expected_array);
+    ASSERT_TRUE(expected->Equals(actual, arrow::EqualOptions::Defaults().diff_sink(&std::cout)))
+        << actual->ToString();
+}
+
+Status CreateTable(const std::string& warehouse, const std::shared_ptr<arrow::Schema>& schema,
+                   const std::vector<std::string>& primary_keys,
+                   const std::map<std::string, std::string>& options) {
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<Catalog> catalog, Catalog::Create(warehouse, options));
+    PAIMON_RETURN_NOT_OK(catalog->CreateDatabase("foo", options, /*ignore_if_exists=*/false));
+    ArrowSchema c_schema;
+    ArrowSchemaMarkReleased(&c_schema);
+    ScopeGuard guard([&c_schema]() { ArrowSchemaRelease(&c_schema); });
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &c_schema));
+    return catalog->CreateTable(Identifier("foo", "bar"), &c_schema,
+                                /*partition_keys=*/{}, primary_keys, options,
+                                /*ignore_if_exists=*/false);
+}
+
+Result<std::unique_ptr<FileIndexFormat::Reader>> CreateFileIndexReader(
+    const std::shared_ptr<DataFileMeta>& data_file, const std::shared_ptr<MemoryPool>& pool) {
+    if (data_file->embedded_index == nullptr) {
+        return Status::Invalid("data file does not contain an embedded file index");
+    }
+    auto input = std::make_shared<ByteArrayInputStream>(data_file->embedded_index->data(),
+                                                        data_file->embedded_index->size());
+    return FileIndexFormat::CreateReader(input, pool);
+}
+
+Result<std::vector<std::shared_ptr<FileIndexReader>>> ReadEmbeddedIndexColumn(
+    const std::shared_ptr<DataFileMeta>& data_file, const std::shared_ptr<arrow::Schema>& schema,
+    const std::string& column, const std::shared_ptr<MemoryPool>& pool) {
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileIndexFormat::Reader> reader,
+                           CreateFileIndexReader(data_file, pool));
+    auto c_schema = std::make_unique<ArrowSchema>();
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, c_schema.get()));
+    return reader->ReadColumnIndex(column, c_schema.get());
+}
+
+class SchemaEvolutionWriteVerifyTest : public ::testing::Test {
+ protected:
+    void SetUp() override {
+        pool_ = GetDefaultPool();
+        dir_ = UniqueTestDirectory::Create("local");
+        ASSERT_TRUE(dir_);
+        table_path_ = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
+    }
+
+    void TearDown() override {
+        dir_.reset();
+    }
+
+    std::shared_ptr<MemoryPool> pool_;
+    std::unique_ptr<UniqueTestDirectory> dir_;
+    std::string table_path_;
+};
+
+TEST_F(SchemaEvolutionWriteVerifyTest,
+       NonRealtimeAppendOldWriterCommitsOldSchemaFileIntoNewSchemaSnapshot) {
+    std::map<std::string, std::string> options = BaseOptions();
+    options["file-index.bitmap.columns"] = "payload";
+    options[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1MB";
+    ASSERT_OK(CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{}, options));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
+                         CreateWriter(table_path_, options));
+
+    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
+                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
+                            DataField(2, EvolvedFields()[2])},
+                           /*highest_field_id=*/2, options));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> old_schema_batch,
+                         MakeBatch(BaseFields(), R"([[1, "old"], [2, "skip"]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK(old_writer->Write(std::move(old_schema_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> messages,
+                         old_writer->PrepareCommit(/*wait_compaction=*/false,
+                                                   /*commit_identifier=*/1));
+    ASSERT_EQ(1, messages.size());
+    std::shared_ptr<CommitMessageImpl> message =
+        std::dynamic_pointer_cast<CommitMessageImpl>(messages[0]);
+    ASSERT_TRUE(message);
+    ASSERT_EQ(1, message->GetNewFilesIncrement().NewFiles().size());
+    std::shared_ptr<DataFileMeta> old_file = message->GetNewFilesIncrement().NewFiles()[0];
+    ASSERT_EQ(0, old_file->schema_id);
+    ASSERT_TRUE(old_file->embedded_index);
+    ASSERT_TRUE(old_file->extra_files.empty());
+    ASSERT_OK_AND_ASSIGN(
+        std::vector<std::shared_ptr<FileIndexReader>> payload_indexes,
+        ReadEmbeddedIndexColumn(old_file, arrow::schema(BaseFields()), "payload", pool_));
+    ASSERT_EQ(1, payload_indexes.size());
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileIndexResult> payload_hit,
+                         payload_indexes[0]->VisitEqual(Literal(FieldType::STRING, "old", 3)));
+    ASSERT_OK_AND_ASSIGN(bool payload_remain, payload_hit->IsRemain());
+    ASSERT_TRUE(payload_remain);
+
+    ASSERT_OK(CommitMessages(table_path_, options, messages, /*commit_identifier=*/1));
+    ASSERT_OK_AND_ASSIGN(std::optional<Snapshot> snapshot,
+                         LatestSnapshot(table_path_, options, dir_->GetFileSystem()));
+    ASSERT_TRUE(snapshot);
+    ASSERT_EQ(1, snapshot->SchemaId());
+
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult all_rows, ReadRows(table_path_, options, pool_));
+    AssertResultEquals(all_rows.data, EvolvedFields(),
+                       R"([[0, 1, "old", null], [0, 2, "skip", null]])");
+
+    auto predicate = PredicateBuilder::Equal(
+        /*field_index=*/1, /*field_name=*/"payload", FieldType::STRING,
+        Literal(FieldType::STRING, "old", 3));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult filtered_rows,
+                         ReadRows(table_path_, options, pool_, /*realtime_context=*/nullptr,
+                                  predicate, /*enable_predicate_filter=*/false));
+    AssertResultEquals(filtered_rows.data, EvolvedFields(), R"([[0, 1, "old", null]])");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest, NonRealtimeAppendNewWriterIndexesNewSchemaColumn) {
+    std::map<std::string, std::string> options = BaseOptions();
+    ASSERT_OK(CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{}, options));
+
+    std::map<std::string, std::string> options_v1 = options;
+    options_v1["file-index.bitmap.columns"] = "extra";
+    options_v1[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1MB";
+    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
+                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
+                            DataField(2, EvolvedFields()[2])},
+                           /*highest_field_id=*/2, options_v1));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
+                         MakeBatch(EvolvedFields(), R"([[1, "new", 20], [2, "skip", 30]])",
+                                   /*partition=*/{}, /*bucket=*/0));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> messages,
+                         WriteWithNewWriter(table_path_, options_v1, std::move(new_schema_batch),
+                                            /*commit_identifier=*/1));
+    ASSERT_EQ(1, messages.size());
+    std::shared_ptr<CommitMessageImpl> message =
+        std::dynamic_pointer_cast<CommitMessageImpl>(messages[0]);
+    ASSERT_TRUE(message);
+    std::shared_ptr<DataFileMeta> new_file = message->GetNewFilesIncrement().NewFiles()[0];
+    ASSERT_EQ(1, new_file->schema_id);
+    ASSERT_TRUE(new_file->embedded_index);
+    ASSERT_TRUE(new_file->extra_files.empty());
+    ASSERT_OK_AND_ASSIGN(
+        std::vector<std::shared_ptr<FileIndexReader>> extra_indexes,
+        ReadEmbeddedIndexColumn(new_file, arrow::schema(EvolvedFields()), "extra", pool_));
+    ASSERT_EQ(1, extra_indexes.size());
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileIndexResult> extra_hit,
+                         extra_indexes[0]->VisitEqual(Literal(20)));
+    ASSERT_OK_AND_ASSIGN(bool extra_remain, extra_hit->IsRemain());
+    ASSERT_TRUE(extra_remain);
+
+    ASSERT_OK(CommitMessages(table_path_, options_v1, messages, /*commit_identifier=*/1));
+    std::shared_ptr<Predicate> predicate = PredicateBuilder::Equal(
+        /*field_index=*/2, /*field_name=*/"extra", FieldType::INT, Literal(20));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows,
+                         ReadRows(table_path_, options_v1, pool_, /*realtime_context=*/nullptr,
+                                  predicate, /*enable_predicate_filter=*/true));
+    AssertResultEquals(rows.data, EvolvedFields(), R"([[0, 1, "new", 20]])");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest, NonRealtimeAppendDataEvolutionWritesPartialNewColumnIndex) {
+    std::map<std::string, std::string> options = DataEvolutionOptions();
+    arrow::FieldVector fields = DataEvolutionFields();
+    ASSERT_OK(CreateTable(dir_->Str(), arrow::schema(fields), /*primary_keys=*/{}, options));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> old_schema_batch,
+                         MakeUnbucketedBatch(fields, R"([[1, "old", "base"]])",
+                                             /*partition=*/{}));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> old_messages,
+                         WriteWithNewWriter(table_path_, options, std::move(old_schema_batch),
+                                            /*commit_identifier=*/1,
+                                            /*write_schema=*/{"f0", "f1", "f2"}));
+    ASSERT_OK(CommitMessages(table_path_, options, old_messages, /*commit_identifier=*/1));
+
+    std::map<std::string, std::string> options_v1 = DataEvolutionOptions();
+    options_v1["file-index.bitmap.columns"] = "f2";
+    options_v1[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1MB";
+    ASSERT_OK(
+        EvolveSchema(table_path_, dir_->GetFileSystem(),
+                     {DataField(0, fields[0]), DataField(1, fields[1]), DataField(2, fields[2])},
+                     /*highest_field_id=*/2, options_v1));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> partial_batch,
+                         MakeUnbucketedBatch({fields[2]}, R"([["updated"]])",
+                                             /*partition=*/{}));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> new_messages,
+                         WriteWithNewWriter(table_path_, options_v1, std::move(partial_batch),
+                                            /*commit_identifier=*/2,
+                                            /*write_schema=*/{"f2"}));
+    ASSERT_EQ(1, new_messages.size());
+    std::shared_ptr<CommitMessageImpl> new_message =
+        std::dynamic_pointer_cast<CommitMessageImpl>(new_messages[0]);
+    ASSERT_TRUE(new_message);
+    std::shared_ptr<DataFileMeta> new_file = new_message->GetNewFilesIncrement().NewFiles()[0];
+    ASSERT_EQ(1, new_file->schema_id);
+    const std::optional<std::vector<std::string>> expected_write_cols =
+        std::vector<std::string>{"f2"};
+    ASSERT_EQ(expected_write_cols, new_file->write_cols);
+    ASSERT_TRUE(new_file->embedded_index);
+    ASSERT_TRUE(new_file->extra_files.empty());
+    ASSERT_OK_AND_ASSIGN(
+        std::vector<std::shared_ptr<FileIndexReader>> f2_indexes,
+        ReadEmbeddedIndexColumn(new_file, arrow::schema({fields[2]}), "f2", pool_));
+    ASSERT_EQ(1, f2_indexes.size());
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileIndexResult> f2_hit,
+                         f2_indexes[0]->VisitEqual(Literal(FieldType::STRING, "updated", 7)));
+    ASSERT_OK_AND_ASSIGN(bool f2_remain, f2_hit->IsRemain());
+    ASSERT_TRUE(f2_remain);
+
+    AssignFirstRowId(new_messages, /*first_row_id=*/0);
+    ASSERT_OK(CommitMessages(table_path_, options_v1, new_messages, /*commit_identifier=*/2));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult all_rows, ReadRows(table_path_, options_v1, pool_));
+    AssertResultEquals(all_rows.data, fields, R"([[0, 1, "old", "updated"]])");
+
+    auto predicate =
+        PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", FieldType::STRING,
+                                Literal(FieldType::STRING, "updated", 7));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult filtered_rows,
+                         ReadRows(table_path_, options_v1, pool_, /*realtime_context=*/nullptr,
+                                  predicate, /*enable_predicate_filter=*/false));
+    AssertResultEquals(filtered_rows.data, fields, R"([[0, 1, "old", "updated"]])");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest, NonRealtimePkOldAndNewSchemaFilesReadThroughLatestSchema) {
+    std::map<std::string, std::string> options = BaseOptions();
+    ASSERT_OK(
+        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
+                         CreateWriter(table_path_, options));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> base_batch,
+                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK(old_writer->Write(std::move(base_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> base_messages,
+                         old_writer->PrepareCommit(/*wait_compaction=*/false,
+                                                   /*commit_identifier=*/1));
+    ASSERT_OK(CommitMessages(table_path_, options, base_messages, /*commit_identifier=*/1));
+
+    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
+                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
+                            DataField(2, EvolvedFields()[2])},
+                           /*highest_field_id=*/2, options));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> new_writer,
+                         CreateWriter(table_path_, options));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
+                         MakeBatch(EvolvedFields(), R"([[2, "new", 20]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK(new_writer->Write(std::move(new_schema_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> new_messages,
+                         new_writer->PrepareCommit(/*wait_compaction=*/false,
+                                                   /*commit_identifier=*/2));
+    ASSERT_EQ(1, new_messages.size());
+    std::shared_ptr<CommitMessageImpl> new_message =
+        std::dynamic_pointer_cast<CommitMessageImpl>(new_messages[0]);
+    ASSERT_TRUE(new_message);
+    ASSERT_EQ(1, new_message->GetNewFilesIncrement().NewFiles().size());
+    ASSERT_EQ(1, new_message->GetNewFilesIncrement().NewFiles()[0]->schema_id);
+    ASSERT_TRUE(new_message->GetNewFilesIncrement().NewFiles()[0]->extra_files.empty());
+
+    ASSERT_OK(CommitMessages(table_path_, options, new_messages, /*commit_identifier=*/2));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> stale_schema_batch,
+                         MakeBatch(BaseFields(), R"([[3, "stale"]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK(old_writer->Write(std::move(stale_schema_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> stale_messages,
+                         old_writer->PrepareCommit(/*wait_compaction=*/false,
+                                                   /*commit_identifier=*/3));
+    ASSERT_EQ(1, stale_messages.size());
+    std::shared_ptr<CommitMessageImpl> stale_message =
+        std::dynamic_pointer_cast<CommitMessageImpl>(stale_messages[0]);
+    ASSERT_TRUE(stale_message);
+    ASSERT_EQ(1, stale_message->GetNewFilesIncrement().NewFiles().size());
+    ASSERT_EQ(0, stale_message->GetNewFilesIncrement().NewFiles()[0]->schema_id);
+    ASSERT_OK(CommitMessages(table_path_, options, stale_messages, /*commit_identifier=*/3));
+
+    ASSERT_OK_AND_ASSIGN(std::optional<Snapshot> snapshot,
+                         LatestSnapshot(table_path_, options, dir_->GetFileSystem()));
+    ASSERT_TRUE(snapshot);
+    ASSERT_EQ(1, snapshot->SchemaId());
+
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows, ReadRows(table_path_, options, pool_));
+    AssertResultEquals(rows.data, EvolvedFields(),
+                       R"([[0, 1, "old", null], [0, 2, "new", 20], [0, 3, "stale", null]])");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest, NonRealtimePkOldWriterCanOverwriteNewColumnWithNull) {
+    std::map<std::string, std::string> options = BaseOptions();
+    ASSERT_OK(
+        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
+                         CreateWriter(table_path_, options));
+
+    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
+                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
+                            DataField(2, EvolvedFields()[2])},
+                           /*highest_field_id=*/2, options));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
+                         MakeBatch(EvolvedFields(), R"([[1, "new", 20]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> new_messages,
+                         WriteWithNewWriter(table_path_, options, std::move(new_schema_batch),
+                                            /*commit_identifier=*/1));
+    ASSERT_OK(CommitMessages(table_path_, options, new_messages, /*commit_identifier=*/1));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> old_schema_batch,
+                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK(old_writer->Write(std::move(old_schema_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> old_messages,
+                         old_writer->PrepareCommit(/*wait_compaction=*/false,
+                                                   /*commit_identifier=*/2));
+    ASSERT_OK(CommitMessages(table_path_, options, old_messages, /*commit_identifier=*/2));
+
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows, ReadRows(table_path_, options, pool_));
+    AssertResultEquals(rows.data, EvolvedFields(), R"([[0, 1, "old", null]])");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest, NonRealtimePkNewWriterIndexesNewSchemaColumn) {
+    std::map<std::string, std::string> options = BaseOptions();
+    ASSERT_OK(
+        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> old_schema_batch,
+                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> old_messages,
+                         WriteWithNewWriter(table_path_, options, std::move(old_schema_batch),
+                                            /*commit_identifier=*/1));
+    ASSERT_OK(CommitMessages(table_path_, options, old_messages, /*commit_identifier=*/1));
+
+    std::map<std::string, std::string> options_v1 = options;
+    options_v1["file-index.bitmap.columns"] = "extra";
+    options_v1[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B";
+    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
+                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
+                            DataField(2, EvolvedFields()[2])},
+                           /*highest_field_id=*/2, options_v1));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
+                         MakeBatch(EvolvedFields(), R"([[2, "new", 20], [3, "skip", 30]])",
+                                   /*partition=*/{}, /*bucket=*/0));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> new_messages,
+                         WriteWithNewWriter(table_path_, options_v1, std::move(new_schema_batch),
+                                            /*commit_identifier=*/2));
+    ASSERT_EQ(1, new_messages.size());
+    std::shared_ptr<CommitMessageImpl> message =
+        std::dynamic_pointer_cast<CommitMessageImpl>(new_messages[0]);
+    ASSERT_TRUE(message);
+    ASSERT_EQ(1, message->GetNewFilesIncrement().NewFiles().size());
+    std::shared_ptr<DataFileMeta> new_file = message->GetNewFilesIncrement().NewFiles()[0];
+    ASSERT_EQ(1, new_file->schema_id);
+    ASSERT_FALSE(new_file->embedded_index);
+    ASSERT_EQ(1, new_file->extra_files.size());
+    ASSERT_TRUE(new_file->extra_files[0]);
+    std::string index_path =
+        PathUtil::JoinPath(table_path_, "bucket-0/" + new_file->extra_files[0].value());
+    ASSERT_OK_AND_ASSIGN(bool index_exists, dir_->GetFileSystem()->Exists(index_path));
+    ASSERT_TRUE(index_exists);
+
+    ASSERT_OK(CommitMessages(table_path_, options_v1, new_messages, /*commit_identifier=*/2));
+    std::shared_ptr<Predicate> predicate = PredicateBuilder::Equal(
+        /*field_index=*/2, /*field_name=*/"extra", FieldType::INT, Literal(20));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows,
+                         ReadRows(table_path_, options_v1, pool_, /*realtime_context=*/nullptr,
+                                  predicate, /*enable_predicate_filter=*/true));
+    AssertResultEquals(rows.data, EvolvedFields(), R"([[0, 2, "new", 20]])");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest, NonRealtimePkEmbeddedFileIndexFailsValueScan) {
+    std::map<std::string, std::string> options = BaseOptions();
+    options["file-index.bitmap.columns"] = "payload";
+    options[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1MB";
+    ASSERT_OK(
+        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         MakeBatch(BaseFields(), R"([[1, "a"], [2, "b"]])",
+                                   /*partition=*/{}, /*bucket=*/0));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> messages,
+                         WriteWithNewWriter(table_path_, options, std::move(batch),
+                                            /*commit_identifier=*/1));
+    ASSERT_EQ(1, messages.size());
+    std::shared_ptr<CommitMessageImpl> message =
+        std::dynamic_pointer_cast<CommitMessageImpl>(messages[0]);
+    ASSERT_TRUE(message);
+    ASSERT_EQ(1, message->GetNewFilesIncrement().NewFiles().size());
+    ASSERT_TRUE(message->GetNewFilesIncrement().NewFiles()[0]->embedded_index);
+    ASSERT_OK(CommitMessages(table_path_, options, messages, /*commit_identifier=*/1));
+
+    std::shared_ptr<Predicate> predicate =
+        PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"payload", FieldType::STRING,
+                                Literal(FieldType::STRING, "a", 1));
+    ASSERT_NOK_WITH_MSG(
+        ScanTable(table_path_, options, pool_, /*realtime_context=*/nullptr, predicate),
+        "do not support embedded index in DataFileMeta");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest, NonRealtimePkSortedIndexConfigDoesNotWriteIndexOnDataWrite) {
+    std::map<std::string, std::string> options = BaseOptions();
+    options[Options::PK_BTREE_INDEX_COLUMNS] = "payload";
+    ASSERT_OK(
+        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         MakeBatch(BaseFields(), R"([[1, "a"], [2, "b"]])",
+                                   /*partition=*/{}, /*bucket=*/0));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> messages,
+                         WriteWithNewWriter(table_path_, options, std::move(batch),
+                                            /*commit_identifier=*/1));
+    ASSERT_EQ(1, messages.size());
+    std::shared_ptr<CommitMessageImpl> message =
+        std::dynamic_pointer_cast<CommitMessageImpl>(messages[0]);
+    ASSERT_TRUE(message);
+    ASSERT_TRUE(message->GetNewFilesIncrement().NewIndexFiles().empty());
+    ASSERT_TRUE(message->GetCompactIncrement().NewIndexFiles().empty());
+    ASSERT_OK(CommitMessages(table_path_, options, messages, /*commit_identifier=*/1));
+    ASSERT_OK_AND_ASSIGN(std::optional<Snapshot> snapshot,
+                         LatestSnapshot(table_path_, options, dir_->GetFileSystem()));
+    ASSERT_TRUE(snapshot);
+    ASSERT_FALSE(snapshot->IndexManifest());
+
+    std::shared_ptr<Predicate> predicate =
+        PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"payload", FieldType::STRING,
+                                Literal(FieldType::STRING, "a", 1));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
+                         ScanTable(table_path_, options, pool_,
+                                   /*realtime_context=*/nullptr, predicate));
+    ASSERT_EQ(0, CountIndexedSplits(plan));
+    std::vector<std::shared_ptr<DataFileMeta>> planned_files = DataFilesFromPlan(plan);
+    ASSERT_EQ(1, planned_files.size());
+    ASSERT_EQ(0, planned_files[0]->schema_id);
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows,
+                         ReadRows(table_path_, options, pool_, /*realtime_context=*/nullptr,
+                                  predicate, /*enable_predicate_filter=*/true));
+    AssertResultEquals(rows.data, BaseFields(), R"([[0, 1, "a"]])");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest, RealtimeAppendRejectsDataEvolutionAtWriterCreation) {
+    std::map<std::string, std::string> create_options = BaseOptions();
+    ASSERT_OK(
+        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{}, create_options));
+
+    std::map<std::string, std::string> write_options = BaseOptions();
+    write_options[Options::DATA_EVOLUTION_ENABLED] = "true";
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_NOK_WITH_MSG(CreateWriter(table_path_, write_options, realtime_context),
+                        "real-time append write does not support data evolution");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest, RealtimeAppendScanRejectsDataEvolutionTable) {
+    std::map<std::string, std::string> options = DataEvolutionOptions();
+    arrow::FieldVector fields = DataEvolutionFields();
+    ASSERT_OK(CreateTable(dir_->Str(), arrow::schema(fields), /*primary_keys=*/{}, options));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         MakeUnbucketedBatch(fields, R"([[1, "old", "base"]])",
+                                             /*partition=*/{}));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> messages,
+                         WriteWithNewWriter(table_path_, options, std::move(batch),
+                                            /*commit_identifier=*/1,
+                                            /*write_schema=*/{"f0", "f1", "f2"}));
+    ASSERT_OK(CommitMessages(table_path_, options, messages, /*commit_identifier=*/1));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+
+    ASSERT_NOK_WITH_MSG(ScanTable(table_path_, options, pool_, realtime_context),
+                        "real-time union read requires fixed bucket mode");
+
+    std::map<std::string, std::string> fixed_bucket_options = options;
+    fixed_bucket_options[Options::BUCKET] = "1";
+    ASSERT_NOK_WITH_MSG(ScanTable(table_path_, fixed_bucket_options, pool_, realtime_context),
+                        "real-time union read does not support data evolution");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest, RealtimeAppendReuseContextKeepsOldMemorySchemaAfterAlter) {
+    std::map<std::string, std::string> options = BaseOptions();
+    ASSERT_OK(CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{}, options));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
+                         CreateWriter(table_path_, options, realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> base_batch,
+                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK(old_writer->Write(std::move(base_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> base_progress,
+                         old_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id,
+                         CommitRealtimeMessages(table_path_, options, base_progress,
+                                                /*commit_identifier=*/1));
+    ASSERT_OK(old_writer->RefreshCommittedSnapshot(snapshot_id));
+
+    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
+                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
+                            DataField(2, EvolvedFields()[2])},
+                           /*highest_field_id=*/2, options));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> reused_context_writer,
+                         CreateWriter(table_path_, options, realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
+                         MakeBatch(EvolvedFields(), R"([[2, "new", 20]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_NOK_WITH_MSG(reused_context_writer->Write(std::move(new_schema_batch)),
+                        "ArrowArray struct has 3 children, expected 2");
+
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows,
+                         ReadRows(table_path_, options, pool_, realtime_context));
+    AssertResultEquals(rows.data, EvolvedFields(), R"([[0, 1, "old", null]])");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest, RealtimeAppendOldWriterAfterAlterCommitsOldSchemaFile) {
+    std::map<std::string, std::string> options = BaseOptions();
+    ASSERT_OK(CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{}, options));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
+                         CreateWriter(table_path_, options, realtime_context));
+
+    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
+                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
+                            DataField(2, EvolvedFields()[2])},
+                           /*highest_field_id=*/2, options));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> old_schema_batch,
+                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK(old_writer->Write(std::move(old_schema_batch)));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult memory_rows,
+                         ReadRows(table_path_, options, pool_, realtime_context));
+    AssertResultEquals(memory_rows.data, EvolvedFields(), R"([[0, 1, "old", null]])");
+
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+                         old_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_EQ(1, progress.size());
+    std::shared_ptr<CommitMessageImpl> message =
+        std::dynamic_pointer_cast<CommitMessageImpl>(progress[0].commit_message);
+    ASSERT_TRUE(message);
+    ASSERT_EQ(1, message->GetNewFilesIncrement().NewFiles().size());
+    ASSERT_EQ(0, message->GetNewFilesIncrement().NewFiles()[0]->schema_id);
+
+    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, CommitRealtimeMessages(table_path_, options, progress,
+                                                                     /*commit_identifier=*/1));
+    ASSERT_OK(old_writer->RefreshCommittedSnapshot(snapshot_id));
+    ASSERT_OK_AND_ASSIGN(std::optional<Snapshot> snapshot,
+                         LatestSnapshot(table_path_, options, dir_->GetFileSystem()));
+    ASSERT_TRUE(snapshot);
+    ASSERT_EQ(1, snapshot->SchemaId());
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult disk_rows, ReadRows(table_path_, options, pool_));
+    AssertResultEquals(disk_rows.data, EvolvedFields(), R"([[0, 1, "old", null]])");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest, RealtimeAppendNewContextUsesNewSchemaAfterAlter) {
+    std::map<std::string, std::string> options = BaseOptions();
+    ASSERT_OK(CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{}, options));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> old_realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
+                         CreateWriter(table_path_, options, old_realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> base_batch,
+                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK(old_writer->Write(std::move(base_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> base_progress,
+                         old_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id,
+                         CommitRealtimeMessages(table_path_, options, base_progress,
+                                                /*commit_identifier=*/1));
+    ASSERT_OK(old_writer->RefreshCommittedSnapshot(snapshot_id));
+
+    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
+                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
+                            DataField(2, EvolvedFields()[2])},
+                           /*highest_field_id=*/2, options));
+
+    std::map<std::string, std::string> options_v1 = options;
+    options_v1["file-index.bitmap.columns"] = "extra";
+    options_v1[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B";
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> new_realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> new_writer,
+                         CreateWriter(table_path_, options_v1, new_realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
+                         MakeBatch(EvolvedFields(), R"([[2, "new", 20]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK(new_writer->Write(std::move(new_schema_batch)));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult memory_rows,
+                         ReadRows(table_path_, options_v1, pool_, new_realtime_context));
+    AssertResultEquals(memory_rows.data, EvolvedFields(),
+                       R"([[0, 1, "old", null], [0, 2, "new", 20]])");
+
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> new_progress,
+                         new_writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
+    ASSERT_EQ(1, new_progress.size());
+    std::shared_ptr<CommitMessageImpl> message =
+        std::dynamic_pointer_cast<CommitMessageImpl>(new_progress[0].commit_message);
+    ASSERT_TRUE(message);
+    ASSERT_EQ(1, message->GetNewFilesIncrement().NewFiles().size());
+    std::shared_ptr<DataFileMeta> new_file = message->GetNewFilesIncrement().NewFiles()[0];
+    ASSERT_EQ(1, new_file->schema_id);
+    ASSERT_FALSE(new_file->embedded_index);
+    ASSERT_EQ(1, new_file->extra_files.size());
+    ASSERT_TRUE(new_file->extra_files[0]);
+    std::string index_path =
+        PathUtil::JoinPath(table_path_, "bucket-0/" + new_file->extra_files[0].value());
+    ASSERT_OK_AND_ASSIGN(bool index_exists, dir_->GetFileSystem()->Exists(index_path));
+    ASSERT_TRUE(index_exists);
+
+    ASSERT_OK_AND_ASSIGN(snapshot_id, CommitRealtimeMessages(table_path_, options_v1, new_progress,
+                                                             /*commit_identifier=*/2));
+    ASSERT_OK(new_writer->RefreshCommittedSnapshot(snapshot_id));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows,
+                         ReadRows(table_path_, options_v1, pool_, new_realtime_context));
+    AssertResultEquals(rows.data, EvolvedFields(), R"([[0, 1, "old", null], [0, 2, "new", 20]])");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest, RealtimePkRejectsDataEvolutionAtWriterCreation) {
+    std::map<std::string, std::string> create_options = BaseOptions();
+    ASSERT_OK(CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"},
+                          create_options));
+
+    std::map<std::string, std::string> write_options = BaseOptions();
+    write_options[Options::DATA_EVOLUTION_ENABLED] = "true";
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_NOK_WITH_MSG(CreateWriter(table_path_, write_options, realtime_context),
+                        "PK realtime v1 does not support data evolution");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest, RealtimePkReuseContextKeepsOldMemorySchemaAfterAlter) {
+    std::map<std::string, std::string> options = BaseOptions();
+    ASSERT_OK(
+        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
+                         CreateWriter(table_path_, options, realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> base_batch,
+                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK(old_writer->Write(std::move(base_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> base_progress,
+                         old_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id,
+                         CommitRealtimeMessages(table_path_, options, base_progress,
+                                                /*commit_identifier=*/1));
+    ASSERT_OK(old_writer->RefreshCommittedSnapshot(snapshot_id));
+
+    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
+                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
+                            DataField(2, EvolvedFields()[2])},
+                           /*highest_field_id=*/2, options));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> reused_context_writer,
+                         CreateWriter(table_path_, options, realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
+                         MakeBatch(EvolvedFields(), R"([[2, "new", 20]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_NOK_WITH_MSG(reused_context_writer->Write(std::move(new_schema_batch)),
+                        "ArrowArray struct has 3 children, expected 2");
+
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows,
+                         ReadRows(table_path_, options, pool_, realtime_context));
+    AssertResultEquals(rows.data, EvolvedFields(), R"([[0, 1, "old", null]])");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest, RealtimePkOldWriterAfterAlterReadsNewColumnAsNull) {
+    std::map<std::string, std::string> options = BaseOptions();
+    ASSERT_OK(
+        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
+                         CreateWriter(table_path_, options, realtime_context));
+
+    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
+                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
+                            DataField(2, EvolvedFields()[2])},
+                           /*highest_field_id=*/2, options));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> old_schema_batch,
+                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK(old_writer->Write(std::move(old_schema_batch)));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult memory_rows,
+                         ReadRows(table_path_, options, pool_, realtime_context));
+    AssertResultEquals(memory_rows.data, EvolvedFields(), R"([[0, 1, "old", null]])");
+
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+                         old_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_EQ(1, progress.size());
+    std::shared_ptr<CommitMessageImpl> message =
+        std::dynamic_pointer_cast<CommitMessageImpl>(progress[0].commit_message);
+    ASSERT_TRUE(message);
+    ASSERT_EQ(1, message->GetNewFilesIncrement().NewFiles().size());
+    ASSERT_EQ(0, message->GetNewFilesIncrement().NewFiles()[0]->schema_id);
+
+    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, CommitRealtimeMessages(table_path_, options, progress,
+                                                                     /*commit_identifier=*/1));
+    ASSERT_OK(old_writer->RefreshCommittedSnapshot(snapshot_id));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult disk_rows, ReadRows(table_path_, options, pool_));
+    AssertResultEquals(disk_rows.data, EvolvedFields(), R"([[0, 1, "old", null]])");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest,
+       RealtimePkOldWriterAfterAlterCannotCommitBehindNewContextOffset) {
+    std::map<std::string, std::string> options = BaseOptions();
+    ASSERT_OK(
+        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
+                         CreateWriter(table_path_, options, realtime_context));
+
+    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
+                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
+                            DataField(2, EvolvedFields()[2])},
+                           /*highest_field_id=*/2, options));
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> new_realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> new_writer,
+                         CreateWriter(table_path_, options, new_realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
+                         MakeBatch(EvolvedFields(), R"([[1, "new", 20]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK(new_writer->Write(std::move(new_schema_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> new_progress,
+                         new_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id,
+                         CommitRealtimeMessages(table_path_, options, new_progress,
+                                                /*commit_identifier=*/1));
+    ASSERT_OK(new_writer->RefreshCommittedSnapshot(snapshot_id));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> old_schema_batch,
+                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK(old_writer->Write(std::move(old_schema_batch)));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult realtime_rows,
+                         ReadRows(table_path_, options, pool_, realtime_context));
+    AssertResultEquals(realtime_rows.data, EvolvedFields(), R"([[0, 1, "new", 20]])");
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> old_progress,
+                         old_writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
+    ASSERT_NOK_WITH_MSG(CommitRealtimeMessages(table_path_, options, old_progress,
+                                               /*commit_identifier=*/2),
+                        "real-time commit offsets for bucket 0 are not contiguous");
+
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult disk_rows, ReadRows(table_path_, options, pool_));
+    AssertResultEquals(disk_rows.data, EvolvedFields(), R"([[0, 1, "new", 20]])");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest, RealtimePkNewContextUsesNewSchemaAfterAlter) {
+    std::map<std::string, std::string> options = BaseOptions();
+    ASSERT_OK(
+        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> old_realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
+                         CreateWriter(table_path_, options, old_realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> base_batch,
+                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK(old_writer->Write(std::move(base_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> base_progress,
+                         old_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id,
+                         CommitRealtimeMessages(table_path_, options, base_progress,
+                                                /*commit_identifier=*/1));
+    ASSERT_OK(old_writer->RefreshCommittedSnapshot(snapshot_id));
+
+    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
+                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
+                            DataField(2, EvolvedFields()[2])},
+                           /*highest_field_id=*/2, options));
+
+    std::map<std::string, std::string> options_v1 = options;
+    options_v1["file-index.bitmap.columns"] = "extra";
+    options_v1[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B";
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> new_realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> new_writer,
+                         CreateWriter(table_path_, options_v1, new_realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
+                         MakeBatch(EvolvedFields(), R"([[2, "new", 20]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK(new_writer->Write(std::move(new_schema_batch)));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult memory_rows,
+                         ReadRows(table_path_, options_v1, pool_, new_realtime_context));
+    AssertResultEquals(memory_rows.data, EvolvedFields(),
+                       R"([[0, 1, "old", null], [0, 2, "new", 20]])");
+
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> new_progress,
+                         new_writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
+    ASSERT_EQ(1, new_progress.size());
+    std::shared_ptr<CommitMessageImpl> message =
+        std::dynamic_pointer_cast<CommitMessageImpl>(new_progress[0].commit_message);
+    ASSERT_TRUE(message);
+    ASSERT_EQ(1, message->GetNewFilesIncrement().NewFiles().size());
+    std::shared_ptr<DataFileMeta> new_file = message->GetNewFilesIncrement().NewFiles()[0];
+    ASSERT_EQ(1, new_file->schema_id);
+    ASSERT_FALSE(new_file->embedded_index);
+    ASSERT_EQ(1, new_file->extra_files.size());
+    ASSERT_TRUE(new_file->extra_files[0]);
+    std::string index_path =
+        PathUtil::JoinPath(table_path_, "bucket-0/" + new_file->extra_files[0].value());
+    ASSERT_OK_AND_ASSIGN(bool index_exists, dir_->GetFileSystem()->Exists(index_path));
+    ASSERT_TRUE(index_exists);
+
+    ASSERT_OK_AND_ASSIGN(snapshot_id, CommitRealtimeMessages(table_path_, options_v1, new_progress,
+                                                             /*commit_identifier=*/2));
+    ASSERT_OK(new_writer->RefreshCommittedSnapshot(snapshot_id));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows,
+                         ReadRows(table_path_, options_v1, pool_, new_realtime_context));
+    AssertResultEquals(rows.data, EvolvedFields(), R"([[0, 1, "old", null], [0, 2, "new", 20]])");
+}
+
+}  // namespace
+}  // namespace paimon::test

From 74d4feee023e53f9bec229de720134e8b16ce89b Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Fri, 21 Aug 2026 14:41:09 +0800
Subject: [PATCH 12/62] fix(realtime): align PK projections by field ID

---
 .../realtime/primary_key_realtime_store.cpp   |   36 +-
 .../primary_key_realtime_store_test.cpp       |   50 +-
 test/inte/CMakeLists.txt                      |    7 -
 ...chema_evolution_write_verify_inte_test.cpp | 1110 -----------------
 4 files changed, 76 insertions(+), 1127 deletions(-)
 delete mode 100644 test/inte/schema_evolution_write_verify_inte_test.cpp

diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index 6565ed8d7..8f51c1b1e 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -98,6 +98,30 @@ uint64_t GetArrayMemoryUsage(const std::shared_ptr<arrow::ArrayData>& data) {
     return result;
 }
 
+int32_t FindPkQueryFieldIndex(const std::shared_ptr<arrow::Schema>& write_schema,
+                              const std::shared_ptr<arrow::Field>& read_field) {
+    Result<int32_t> read_id = NestedProjectionUtils::GetPaimonFieldId(read_field);
+    if (read_id.ok()) {
+        Result<std::shared_ptr<arrow::Field>> write_field =
+            NestedProjectionUtils::FindFieldByPaimonId(arrow::struct_(write_schema->fields()),
+                                                       read_id.value());
+        if (write_field.ok()) {
+            return write_schema->GetFieldIndex(write_field.value()->name());
+        }
+    }
+
+    const int32_t name_index = write_schema->GetFieldIndex(read_field->name());
+    if (name_index < 0) {
+        return -1;
+    }
+    Result<int32_t> write_id =
+        NestedProjectionUtils::GetPaimonFieldId(write_schema->field(name_index));
+    if (read_id.ok() && write_id.ok() && read_id.value() != write_id.value()) {
+        return -1;
+    }
+    return name_index;
+}
+
 struct StoredBatch {
     std::shared_ptr<arrow::StructArray> data;
     std::vector<RecordBatch::RowKind> row_kinds;
@@ -423,17 +447,23 @@ class PrimaryKeyRealtimeStore::Impl {
                 projection.push_back(KeyValueProjectionConsumer::kSequenceNumberProjection);
                 continue;
             }
-            int32_t index = write_schema_->GetFieldIndex(field->name());
+            int32_t index = FindPkQueryFieldIndex(write_schema_, field);
             if (index < 0) {
                 Result<int32_t> field_id = NestedProjectionUtils::GetPaimonFieldId(field);
                 if (!field_id.ok()) {
                     return Status::Invalid(
                         "PK real-time query field is missing from write schema: ", field->name());
                 }
+                std::string internal_name =
+                    "__paimon_pk_realtime_null_" + std::to_string(field_id.value());
+                while (
+                    NestedProjectionUtils::FindFieldByName(aligned_value_fields, internal_name)) {
+                    internal_name.push_back('_');
+                }
                 index = static_cast<int32_t>(aligned_value_fields.size());
-                aligned_value_fields.push_back(field);
+                aligned_value_fields.push_back(field->WithName(internal_name));
             } else {
-                aligned_value_fields[index] = field;
+                aligned_value_fields[index] = write_schema_->field(index)->WithType(field->type());
             }
             projection.push_back(index);
         }
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index ef293e54b..66901a6b1 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -392,6 +392,40 @@ TEST_F(PrimaryKeyRealtimeStoreTest, TestQueryProjection) {
                         "query field is missing from write schema: unknown");
 }
 
+TEST_F(PrimaryKeyRealtimeStoreTest, TestQueryProjectionMatchesRenamedFieldsById) {
+    const std::shared_ptr<arrow::Field> id =
+        DataField::ConvertDataFieldToArrowField(DataField(0, arrow::field("id", arrow::int64())));
+    const std::shared_ptr<arrow::Field> value =
+        DataField::ConvertDataFieldToArrowField(DataField(1, arrow::field("value", arrow::utf8())));
+    const std::shared_ptr<arrow::Schema> write_schema = arrow::schema({id, value});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                         CreateStore(write_schema, {"id"}, /*restore_max_sequence=*/4));
+    ASSERT_OK(store->Write(
+        RealtimeWriteBatch{MakeBatch(R"([[1, "kept"]])", {}, write_schema), OffsetRange(0, 1)}));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
+
+    const std::shared_ptr<arrow::Field> renamed_value = DataField::ConvertDataFieldToArrowField(
+        DataField(1, arrow::field("renamed", arrow::utf8())));
+    const std::shared_ptr<arrow::Field> renamed_id = DataField::ConvertDataFieldToArrowField(
+        DataField(0, arrow::field("renamed_id", arrow::int64())));
+    const std::shared_ptr<arrow::Field> replaced =
+        DataField::ConvertDataFieldToArrowField(DataField(2, arrow::field("value", arrow::utf8())));
+    const std::shared_ptr<arrow::Field> replaced_id =
+        DataField::ConvertDataFieldToArrowField(DataField(4, arrow::field("id", arrow::int64())));
+    const std::shared_ptr<arrow::Field> added =
+        DataField::ConvertDataFieldToArrowField(DataField(3, arrow::field("added", arrow::utf8())));
+    std::unique_ptr<ArrowSchema> read_schema =
+        MakeReadSchema({renamed_value, renamed_id, replaced, replaced_id, added});
+    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store->CreateQueryReaders(view, /*offset_begin=*/0, context));
+    const std::shared_ptr<arrow::DataType> result_type =
+        arrow::struct_({DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind()),
+                        renamed_value, renamed_id, replaced, replaced_id, added});
+    AssertReaderOutput(readers, result_type, R"([[0, "kept", 1, null, null, null]])");
+}
+
 TEST_F(PrimaryKeyRealtimeStoreTest, TestNestedProjection) {
     const std::shared_ptr<arrow::Field> id =
         DataField::ConvertDataFieldToArrowField(DataField(0, arrow::field("id", arrow::int64())));
@@ -433,7 +467,10 @@ TEST_F(PrimaryKeyRealtimeStoreTest, TestCompositeKeyClipping) {
                                      {}, composite_schema),
                            OffsetRange(20, 24)}));
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
-    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(FullQueryFields(composite_schema));
+    const std::shared_ptr<arrow::Field> sequence =
+        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber());
+    std::unique_ptr<ArrowSchema> read_schema =
+        MakeReadSchema({sequence, composite_schema->field(0), composite_schema->field(2)});
     RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
                                  /*enable_predicate_pushdown=*/false};
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
@@ -445,13 +482,12 @@ TEST_F(PrimaryKeyRealtimeStoreTest, TestCompositeKeyClipping) {
     ASSERT_EQ("c", range->GetMinKey()->GetString(1).ToString());
     ASSERT_EQ(2, range->GetMaxKey()->GetLong(0));
     ASSERT_EQ("b", range->GetMaxKey()->GetString(1).ToString());
-    std::shared_ptr<arrow::DataType> query_type = arrow::struct_(
-        {DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind()),
-         DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber()),
-         composite_schema->field(0), composite_schema->field(1), composite_schema->field(2)});
+    std::shared_ptr<arrow::DataType> query_type =
+        arrow::struct_({DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind()),
+                        sequence, composite_schema->field(0), composite_schema->field(2)});
     AssertReaderOutput(readers, query_type,
-                       R"([[0, 7, 1, "c", "one-c"], [0, 8, 2, "a", "two-a"],
-                            [0, 6, 2, "b", "two-b"]])");
+                       R"([[0, 7, 1, "one-c"], [0, 8, 2, "two-a"],
+                            [0, 6, 2, "two-b"]])");
 }
 
 }  // namespace paimon::test
diff --git a/test/inte/CMakeLists.txt b/test/inte/CMakeLists.txt
index f1b3f8ce6..75147ce60 100644
--- a/test/inte/CMakeLists.txt
+++ b/test/inte/CMakeLists.txt
@@ -43,13 +43,6 @@ if(PAIMON_BUILD_TESTS)
                     test_utils_static
                     ${GTEST_LINK_TOOLCHAIN})
 
-    add_paimon_test(schema_evolution_write_verify_inte_test
-                    STATIC_LINK_LIBS
-                    paimon_shared
-                    ${TEST_STATIC_LINK_LIBS}
-                    test_utils_static
-                    ${GTEST_LINK_TOOLCHAIN})
-
     add_paimon_test(global_index_test
                     STATIC_LINK_LIBS
                     paimon_shared
diff --git a/test/inte/schema_evolution_write_verify_inte_test.cpp b/test/inte/schema_evolution_write_verify_inte_test.cpp
deleted file mode 100644
index dcadbd9e1..000000000
--- a/test/inte/schema_evolution_write_verify_inte_test.cpp
+++ /dev/null
@@ -1,1110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <cstddef>
-#include <cstdint>
-#include <map>
-#include <memory>
-#include <optional>
-#include <string>
-#include <utility>
-#include <vector>
-
-#include "arrow/api.h"
-#include "arrow/c/bridge.h"
-#include "arrow/ipc/json_simple.h"
-#include "gtest/gtest.h"
-#include "paimon/catalog/catalog.h"
-#include "paimon/catalog/identifier.h"
-#include "paimon/commit_context.h"
-#include "paimon/common/utils/path_util.h"
-#include "paimon/common/utils/scope_guard.h"
-#include "paimon/core/core_options.h"
-#include "paimon/core/global_index/indexed_split_impl.h"
-#include "paimon/core/io/data_file_meta.h"
-#include "paimon/core/schema/schema_manager.h"
-#include "paimon/core/snapshot.h"
-#include "paimon/core/table/sink/commit_message_impl.h"
-#include "paimon/core/table/source/data_split_impl.h"
-#include "paimon/defs.h"
-#include "paimon/file_index/file_index_format.h"
-#include "paimon/file_index/file_index_reader.h"
-#include "paimon/file_index/file_index_result.h"
-#include "paimon/file_store_commit.h"
-#include "paimon/file_store_write.h"
-#include "paimon/fs/file_system.h"
-#include "paimon/io/byte_array_input_stream.h"
-#include "paimon/predicate/literal.h"
-#include "paimon/predicate/predicate_builder.h"
-#include "paimon/read_context.h"
-#include "paimon/reader/batch_reader.h"
-#include "paimon/realtime/realtime_context.h"
-#include "paimon/record_batch.h"
-#include "paimon/scan_context.h"
-#include "paimon/table/source/plan.h"
-#include "paimon/table/source/startup_mode.h"
-#include "paimon/table/source/table_read.h"
-#include "paimon/table/source/table_scan.h"
-#include "paimon/testing/utils/read_result_collector.h"
-#include "paimon/testing/utils/test_helper.h"
-#include "paimon/testing/utils/testharness.h"
-#include "paimon/write_context.h"
-
-namespace paimon::test {
-namespace {
-
-std::map<std::string, std::string> BaseOptions() {
-    return {{Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, "orc"},
-            {Options::FILE_SYSTEM, "local"},   {Options::BUCKET, "1"},
-            {Options::BUCKET_KEY, "id"},       {Options::TARGET_FILE_SIZE, "1MB"}};
-}
-
-std::map<std::string, std::string> DataEvolutionOptions() {
-    return {{Options::MANIFEST_FORMAT, "orc"},       {Options::FILE_FORMAT, "orc"},
-            {Options::FILE_SYSTEM, "local"},         {Options::TARGET_FILE_SIZE, "1MB"},
-            {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}};
-}
-
-arrow::FieldVector BaseFields() {
-    return {arrow::field("id", arrow::int64()), arrow::field("payload", arrow::utf8())};
-}
-
-arrow::FieldVector EvolvedFields() {
-    return {arrow::field("id", arrow::int64()), arrow::field("payload", arrow::utf8()),
-            arrow::field("extra", arrow::int32())};
-}
-
-arrow::FieldVector DataEvolutionFields() {
-    return {arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::utf8()),
-            arrow::field("f2", arrow::utf8())};
-}
-
-Result<std::unique_ptr<RecordBatch>> MakeBatch(
-    const arrow::FieldVector& fields, const std::string& json,
-    const std::map<std::string, std::string>& partition, int32_t bucket,
-    const std::vector<RecordBatch::RowKind>& row_kinds = {}) {
-    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-        std::shared_ptr<arrow::Array> array,
-        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), json));
-    ArrowArray c_array;
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array));
-    RecordBatchBuilder builder(&c_array);
-    return builder.SetPartition(partition).SetBucket(bucket).SetRowKinds(row_kinds).Finish();
-}
-
-Result<std::unique_ptr<RecordBatch>> MakeUnbucketedBatch(
-    const arrow::FieldVector& fields, const std::string& json,
-    const std::map<std::string, std::string>& partition,
-    const std::vector<RecordBatch::RowKind>& row_kinds = {}) {
-    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-        std::shared_ptr<arrow::Array> array,
-        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), json));
-    ArrowArray c_array;
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array));
-    RecordBatchBuilder builder(&c_array);
-    return builder.SetPartition(partition).SetRowKinds(row_kinds).Finish();
-}
-
-Result<std::unique_ptr<FileStoreWrite>> CreateWriter(
-    const std::string& table_path, const std::map<std::string, std::string>& options,
-    const std::shared_ptr<RealtimeContext>& realtime_context = nullptr,
-    const std::vector<std::string>& write_schema = {}) {
-    WriteContextBuilder builder(table_path, "schema_evolution_verify");
-    builder.SetOptions(options).WithStreamingMode(true);
-    if (realtime_context) {
-        builder.WithRealtimeContext(realtime_context);
-    }
-    if (!write_schema.empty()) {
-        builder.WithWriteSchema(write_schema);
-    }
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<WriteContext> context, builder.Finish());
-    return FileStoreWrite::Create(std::move(context));
-}
-
-Result<std::vector<std::shared_ptr<CommitMessage>>> WriteWithNewWriter(
-    const std::string& table_path, const std::map<std::string, std::string>& options,
-    std::unique_ptr<RecordBatch> batch, int64_t commit_identifier,
-    const std::vector<std::string>& write_schema = {}) {
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreWrite> writer,
-                           CreateWriter(table_path, options, nullptr, write_schema));
-    PAIMON_RETURN_NOT_OK(writer->Write(std::move(batch)));
-    PAIMON_ASSIGN_OR_RAISE(std::vector<std::shared_ptr<CommitMessage>> messages,
-                           writer->PrepareCommit(/*wait_compaction=*/false, commit_identifier));
-    PAIMON_RETURN_NOT_OK(writer->Close());
-    return messages;
-}
-
-Result<std::unique_ptr<FileStoreCommit>> CreateCommit(
-    const std::string& table_path, const std::map<std::string, std::string>& options) {
-    CommitContextBuilder builder(table_path, "schema_evolution_verify");
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<CommitContext> context,
-                           builder.SetOptions(options).IgnoreEmptyCommit(false).Finish());
-    return FileStoreCommit::Create(std::move(context));
-}
-
-Status CommitMessages(const std::string& table_path,
-                      const std::map<std::string, std::string>& options,
-                      const std::vector<std::shared_ptr<CommitMessage>>& messages,
-                      int64_t commit_identifier) {
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreCommit> commit,
-                           CreateCommit(table_path, options));
-    return commit->Commit(messages, commit_identifier);
-}
-
-Result<int64_t> CommitRealtimeMessages(const std::string& table_path,
-                                       const std::map<std::string, std::string>& options,
-                                       const std::vector<RealtimeCommitProgress>& messages,
-                                       int64_t commit_identifier) {
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreCommit> commit,
-                           CreateCommit(table_path, options));
-    return commit->CommitWithProgress(messages, commit_identifier, /*watermark=*/std::nullopt);
-}
-
-Result<std::optional<Snapshot>> LatestSnapshot(const std::string& table_path,
-                                               const std::map<std::string, std::string>& options,
-                                               const std::shared_ptr<FileSystem>& file_system) {
-    PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options, file_system));
-    SnapshotManager snapshot_manager(core_options.GetFileSystem(), table_path);
-    return snapshot_manager.LatestSnapshot();
-}
-
-Result<std::shared_ptr<Plan>> ScanTable(
-    const std::string& table_path, const std::map<std::string, std::string>& options,
-    const std::shared_ptr<MemoryPool>& pool,
-    const std::shared_ptr<RealtimeContext>& realtime_context = nullptr,
-    const std::shared_ptr<Predicate>& predicate = nullptr) {
-    ScanContextBuilder scan_builder(table_path);
-    scan_builder.SetOptions(options)
-        .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString())
-        .SetPredicate(predicate)
-        .WithMemoryPool(pool);
-    if (realtime_context) {
-        scan_builder.WithRealtimeContext(realtime_context);
-    }
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ScanContext> scan_context, scan_builder.Finish());
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableScan> table_scan,
-                           TableScan::Create(std::move(scan_context)));
-    return table_scan->CreatePlan();
-}
-
-std::vector<std::shared_ptr<DataFileMeta>> DataFilesFromPlan(const std::shared_ptr<Plan>& plan) {
-    std::vector<std::shared_ptr<DataFileMeta>> files;
-    for (const std::shared_ptr<Split>& split : plan->Splits()) {
-        std::shared_ptr<Split> data_split = split;
-        if (std::shared_ptr<IndexedSplitImpl> indexed_split =
-                std::dynamic_pointer_cast<IndexedSplitImpl>(split)) {
-            data_split = indexed_split->GetDataSplit();
-        }
-        std::shared_ptr<DataSplitImpl> split_impl =
-            std::dynamic_pointer_cast<DataSplitImpl>(data_split);
-        if (!split_impl) {
-            continue;
-        }
-        const std::vector<std::shared_ptr<DataFileMeta>>& split_files = split_impl->DataFiles();
-        files.insert(files.end(), split_files.begin(), split_files.end());
-    }
-    return files;
-}
-
-size_t CountIndexedSplits(const std::shared_ptr<Plan>& plan) {
-    size_t count = 0;
-    for (const std::shared_ptr<Split>& split : plan->Splits()) {
-        if (std::dynamic_pointer_cast<IndexedSplitImpl>(split)) {
-            count++;
-        }
-    }
-    return count;
-}
-
-Status EvolveSchema(const std::string& table_path, const std::shared_ptr<FileSystem>& file_system,
-                    const std::vector<DataField>& fields, int32_t highest_field_id,
-                    const std::map<std::string, std::string>& options) {
-    return TestHelper::WriteNextSchema(file_system, table_path, fields, highest_field_id, options);
-}
-
-void AssignFirstRowId(const std::vector<std::shared_ptr<CommitMessage>>& messages,
-                      int64_t first_row_id) {
-    for (const std::shared_ptr<CommitMessage>& commit_message : messages) {
-        std::shared_ptr<CommitMessageImpl> message =
-            std::dynamic_pointer_cast<CommitMessageImpl>(commit_message);
-        ASSERT_TRUE(message);
-        for (const std::shared_ptr<DataFileMeta>& file :
-             message->GetNewFilesIncrement().NewFiles()) {
-            file->AssignFirstRowId(first_row_id);
-        }
-    }
-}
-
-struct CollectedReadResult {
-    std::unique_ptr<TableRead> table_read;
-    std::unique_ptr<BatchReader> reader;
-    std::shared_ptr<arrow::ChunkedArray> data;
-};
-
-Result<CollectedReadResult> ReadRows(
-    const std::string& table_path, const std::map<std::string, std::string>& options,
-    const std::shared_ptr<MemoryPool>& pool,
-    const std::shared_ptr<RealtimeContext>& realtime_context = nullptr,
-    const std::shared_ptr<Predicate>& predicate = nullptr, bool enable_predicate_filter = true) {
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Plan> plan,
-                           ScanTable(table_path, options, pool, realtime_context, predicate));
-
-    ReadContextBuilder read_builder(table_path);
-    read_builder.SetOptions(options)
-        .SetPredicate(predicate)
-        .EnablePredicateFilter(enable_predicate_filter)
-        .WithMemoryPool(pool);
-    if (realtime_context) {
-        read_builder.WithRealtimeContext(realtime_context);
-    }
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableRead> table_read,
-                           TableRead::Create(std::move(read_context)));
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> batch_reader,
-                           table_read->CreateReader(plan->Splits()));
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::ChunkedArray> rows,
-                           ReadResultCollector::CollectResult(batch_reader.get()));
-    return CollectedReadResult{std::move(table_read), std::move(batch_reader), std::move(rows)};
-}
-
-void AssertResultEquals(const std::shared_ptr<arrow::ChunkedArray>& actual,
-                        const arrow::FieldVector& fields, const std::string& expected_json) {
-    arrow::FieldVector fields_with_row_kind = fields;
-    fields_with_row_kind.insert(fields_with_row_kind.begin(),
-                                arrow::field("_VALUE_KIND", arrow::int8()));
-    std::shared_ptr<arrow::Array> expected_array =
-        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_with_row_kind),
-                                                  expected_json)
-            .ValueOrDie();
-    auto expected = std::make_shared<arrow::ChunkedArray>(expected_array);
-    ASSERT_TRUE(expected->Equals(actual, arrow::EqualOptions::Defaults().diff_sink(&std::cout)))
-        << actual->ToString();
-}
-
-Status CreateTable(const std::string& warehouse, const std::shared_ptr<arrow::Schema>& schema,
-                   const std::vector<std::string>& primary_keys,
-                   const std::map<std::string, std::string>& options) {
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<Catalog> catalog, Catalog::Create(warehouse, options));
-    PAIMON_RETURN_NOT_OK(catalog->CreateDatabase("foo", options, /*ignore_if_exists=*/false));
-    ArrowSchema c_schema;
-    ArrowSchemaMarkReleased(&c_schema);
-    ScopeGuard guard([&c_schema]() { ArrowSchemaRelease(&c_schema); });
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &c_schema));
-    return catalog->CreateTable(Identifier("foo", "bar"), &c_schema,
-                                /*partition_keys=*/{}, primary_keys, options,
-                                /*ignore_if_exists=*/false);
-}
-
-Result<std::unique_ptr<FileIndexFormat::Reader>> CreateFileIndexReader(
-    const std::shared_ptr<DataFileMeta>& data_file, const std::shared_ptr<MemoryPool>& pool) {
-    if (data_file->embedded_index == nullptr) {
-        return Status::Invalid("data file does not contain an embedded file index");
-    }
-    auto input = std::make_shared<ByteArrayInputStream>(data_file->embedded_index->data(),
-                                                        data_file->embedded_index->size());
-    return FileIndexFormat::CreateReader(input, pool);
-}
-
-Result<std::vector<std::shared_ptr<FileIndexReader>>> ReadEmbeddedIndexColumn(
-    const std::shared_ptr<DataFileMeta>& data_file, const std::shared_ptr<arrow::Schema>& schema,
-    const std::string& column, const std::shared_ptr<MemoryPool>& pool) {
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileIndexFormat::Reader> reader,
-                           CreateFileIndexReader(data_file, pool));
-    auto c_schema = std::make_unique<ArrowSchema>();
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, c_schema.get()));
-    return reader->ReadColumnIndex(column, c_schema.get());
-}
-
-class SchemaEvolutionWriteVerifyTest : public ::testing::Test {
- protected:
-    void SetUp() override {
-        pool_ = GetDefaultPool();
-        dir_ = UniqueTestDirectory::Create("local");
-        ASSERT_TRUE(dir_);
-        table_path_ = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
-    }
-
-    void TearDown() override {
-        dir_.reset();
-    }
-
-    std::shared_ptr<MemoryPool> pool_;
-    std::unique_ptr<UniqueTestDirectory> dir_;
-    std::string table_path_;
-};
-
-TEST_F(SchemaEvolutionWriteVerifyTest,
-       NonRealtimeAppendOldWriterCommitsOldSchemaFileIntoNewSchemaSnapshot) {
-    std::map<std::string, std::string> options = BaseOptions();
-    options["file-index.bitmap.columns"] = "payload";
-    options[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1MB";
-    ASSERT_OK(CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{}, options));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
-                         CreateWriter(table_path_, options));
-
-    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
-                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
-                            DataField(2, EvolvedFields()[2])},
-                           /*highest_field_id=*/2, options));
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> old_schema_batch,
-                         MakeBatch(BaseFields(), R"([[1, "old"], [2, "skip"]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK(old_writer->Write(std::move(old_schema_batch)));
-    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> messages,
-                         old_writer->PrepareCommit(/*wait_compaction=*/false,
-                                                   /*commit_identifier=*/1));
-    ASSERT_EQ(1, messages.size());
-    std::shared_ptr<CommitMessageImpl> message =
-        std::dynamic_pointer_cast<CommitMessageImpl>(messages[0]);
-    ASSERT_TRUE(message);
-    ASSERT_EQ(1, message->GetNewFilesIncrement().NewFiles().size());
-    std::shared_ptr<DataFileMeta> old_file = message->GetNewFilesIncrement().NewFiles()[0];
-    ASSERT_EQ(0, old_file->schema_id);
-    ASSERT_TRUE(old_file->embedded_index);
-    ASSERT_TRUE(old_file->extra_files.empty());
-    ASSERT_OK_AND_ASSIGN(
-        std::vector<std::shared_ptr<FileIndexReader>> payload_indexes,
-        ReadEmbeddedIndexColumn(old_file, arrow::schema(BaseFields()), "payload", pool_));
-    ASSERT_EQ(1, payload_indexes.size());
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileIndexResult> payload_hit,
-                         payload_indexes[0]->VisitEqual(Literal(FieldType::STRING, "old", 3)));
-    ASSERT_OK_AND_ASSIGN(bool payload_remain, payload_hit->IsRemain());
-    ASSERT_TRUE(payload_remain);
-
-    ASSERT_OK(CommitMessages(table_path_, options, messages, /*commit_identifier=*/1));
-    ASSERT_OK_AND_ASSIGN(std::optional<Snapshot> snapshot,
-                         LatestSnapshot(table_path_, options, dir_->GetFileSystem()));
-    ASSERT_TRUE(snapshot);
-    ASSERT_EQ(1, snapshot->SchemaId());
-
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult all_rows, ReadRows(table_path_, options, pool_));
-    AssertResultEquals(all_rows.data, EvolvedFields(),
-                       R"([[0, 1, "old", null], [0, 2, "skip", null]])");
-
-    auto predicate = PredicateBuilder::Equal(
-        /*field_index=*/1, /*field_name=*/"payload", FieldType::STRING,
-        Literal(FieldType::STRING, "old", 3));
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult filtered_rows,
-                         ReadRows(table_path_, options, pool_, /*realtime_context=*/nullptr,
-                                  predicate, /*enable_predicate_filter=*/false));
-    AssertResultEquals(filtered_rows.data, EvolvedFields(), R"([[0, 1, "old", null]])");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest, NonRealtimeAppendNewWriterIndexesNewSchemaColumn) {
-    std::map<std::string, std::string> options = BaseOptions();
-    ASSERT_OK(CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{}, options));
-
-    std::map<std::string, std::string> options_v1 = options;
-    options_v1["file-index.bitmap.columns"] = "extra";
-    options_v1[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1MB";
-    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
-                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
-                            DataField(2, EvolvedFields()[2])},
-                           /*highest_field_id=*/2, options_v1));
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
-                         MakeBatch(EvolvedFields(), R"([[1, "new", 20], [2, "skip", 30]])",
-                                   /*partition=*/{}, /*bucket=*/0));
-    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> messages,
-                         WriteWithNewWriter(table_path_, options_v1, std::move(new_schema_batch),
-                                            /*commit_identifier=*/1));
-    ASSERT_EQ(1, messages.size());
-    std::shared_ptr<CommitMessageImpl> message =
-        std::dynamic_pointer_cast<CommitMessageImpl>(messages[0]);
-    ASSERT_TRUE(message);
-    std::shared_ptr<DataFileMeta> new_file = message->GetNewFilesIncrement().NewFiles()[0];
-    ASSERT_EQ(1, new_file->schema_id);
-    ASSERT_TRUE(new_file->embedded_index);
-    ASSERT_TRUE(new_file->extra_files.empty());
-    ASSERT_OK_AND_ASSIGN(
-        std::vector<std::shared_ptr<FileIndexReader>> extra_indexes,
-        ReadEmbeddedIndexColumn(new_file, arrow::schema(EvolvedFields()), "extra", pool_));
-    ASSERT_EQ(1, extra_indexes.size());
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileIndexResult> extra_hit,
-                         extra_indexes[0]->VisitEqual(Literal(20)));
-    ASSERT_OK_AND_ASSIGN(bool extra_remain, extra_hit->IsRemain());
-    ASSERT_TRUE(extra_remain);
-
-    ASSERT_OK(CommitMessages(table_path_, options_v1, messages, /*commit_identifier=*/1));
-    std::shared_ptr<Predicate> predicate = PredicateBuilder::Equal(
-        /*field_index=*/2, /*field_name=*/"extra", FieldType::INT, Literal(20));
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows,
-                         ReadRows(table_path_, options_v1, pool_, /*realtime_context=*/nullptr,
-                                  predicate, /*enable_predicate_filter=*/true));
-    AssertResultEquals(rows.data, EvolvedFields(), R"([[0, 1, "new", 20]])");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest, NonRealtimeAppendDataEvolutionWritesPartialNewColumnIndex) {
-    std::map<std::string, std::string> options = DataEvolutionOptions();
-    arrow::FieldVector fields = DataEvolutionFields();
-    ASSERT_OK(CreateTable(dir_->Str(), arrow::schema(fields), /*primary_keys=*/{}, options));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> old_schema_batch,
-                         MakeUnbucketedBatch(fields, R"([[1, "old", "base"]])",
-                                             /*partition=*/{}));
-    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> old_messages,
-                         WriteWithNewWriter(table_path_, options, std::move(old_schema_batch),
-                                            /*commit_identifier=*/1,
-                                            /*write_schema=*/{"f0", "f1", "f2"}));
-    ASSERT_OK(CommitMessages(table_path_, options, old_messages, /*commit_identifier=*/1));
-
-    std::map<std::string, std::string> options_v1 = DataEvolutionOptions();
-    options_v1["file-index.bitmap.columns"] = "f2";
-    options_v1[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1MB";
-    ASSERT_OK(
-        EvolveSchema(table_path_, dir_->GetFileSystem(),
-                     {DataField(0, fields[0]), DataField(1, fields[1]), DataField(2, fields[2])},
-                     /*highest_field_id=*/2, options_v1));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> partial_batch,
-                         MakeUnbucketedBatch({fields[2]}, R"([["updated"]])",
-                                             /*partition=*/{}));
-    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> new_messages,
-                         WriteWithNewWriter(table_path_, options_v1, std::move(partial_batch),
-                                            /*commit_identifier=*/2,
-                                            /*write_schema=*/{"f2"}));
-    ASSERT_EQ(1, new_messages.size());
-    std::shared_ptr<CommitMessageImpl> new_message =
-        std::dynamic_pointer_cast<CommitMessageImpl>(new_messages[0]);
-    ASSERT_TRUE(new_message);
-    std::shared_ptr<DataFileMeta> new_file = new_message->GetNewFilesIncrement().NewFiles()[0];
-    ASSERT_EQ(1, new_file->schema_id);
-    const std::optional<std::vector<std::string>> expected_write_cols =
-        std::vector<std::string>{"f2"};
-    ASSERT_EQ(expected_write_cols, new_file->write_cols);
-    ASSERT_TRUE(new_file->embedded_index);
-    ASSERT_TRUE(new_file->extra_files.empty());
-    ASSERT_OK_AND_ASSIGN(
-        std::vector<std::shared_ptr<FileIndexReader>> f2_indexes,
-        ReadEmbeddedIndexColumn(new_file, arrow::schema({fields[2]}), "f2", pool_));
-    ASSERT_EQ(1, f2_indexes.size());
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileIndexResult> f2_hit,
-                         f2_indexes[0]->VisitEqual(Literal(FieldType::STRING, "updated", 7)));
-    ASSERT_OK_AND_ASSIGN(bool f2_remain, f2_hit->IsRemain());
-    ASSERT_TRUE(f2_remain);
-
-    AssignFirstRowId(new_messages, /*first_row_id=*/0);
-    ASSERT_OK(CommitMessages(table_path_, options_v1, new_messages, /*commit_identifier=*/2));
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult all_rows, ReadRows(table_path_, options_v1, pool_));
-    AssertResultEquals(all_rows.data, fields, R"([[0, 1, "old", "updated"]])");
-
-    auto predicate =
-        PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", FieldType::STRING,
-                                Literal(FieldType::STRING, "updated", 7));
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult filtered_rows,
-                         ReadRows(table_path_, options_v1, pool_, /*realtime_context=*/nullptr,
-                                  predicate, /*enable_predicate_filter=*/false));
-    AssertResultEquals(filtered_rows.data, fields, R"([[0, 1, "old", "updated"]])");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest, NonRealtimePkOldAndNewSchemaFilesReadThroughLatestSchema) {
-    std::map<std::string, std::string> options = BaseOptions();
-    ASSERT_OK(
-        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
-                         CreateWriter(table_path_, options));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> base_batch,
-                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK(old_writer->Write(std::move(base_batch)));
-    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> base_messages,
-                         old_writer->PrepareCommit(/*wait_compaction=*/false,
-                                                   /*commit_identifier=*/1));
-    ASSERT_OK(CommitMessages(table_path_, options, base_messages, /*commit_identifier=*/1));
-
-    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
-                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
-                            DataField(2, EvolvedFields()[2])},
-                           /*highest_field_id=*/2, options));
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> new_writer,
-                         CreateWriter(table_path_, options));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
-                         MakeBatch(EvolvedFields(), R"([[2, "new", 20]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK(new_writer->Write(std::move(new_schema_batch)));
-    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> new_messages,
-                         new_writer->PrepareCommit(/*wait_compaction=*/false,
-                                                   /*commit_identifier=*/2));
-    ASSERT_EQ(1, new_messages.size());
-    std::shared_ptr<CommitMessageImpl> new_message =
-        std::dynamic_pointer_cast<CommitMessageImpl>(new_messages[0]);
-    ASSERT_TRUE(new_message);
-    ASSERT_EQ(1, new_message->GetNewFilesIncrement().NewFiles().size());
-    ASSERT_EQ(1, new_message->GetNewFilesIncrement().NewFiles()[0]->schema_id);
-    ASSERT_TRUE(new_message->GetNewFilesIncrement().NewFiles()[0]->extra_files.empty());
-
-    ASSERT_OK(CommitMessages(table_path_, options, new_messages, /*commit_identifier=*/2));
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> stale_schema_batch,
-                         MakeBatch(BaseFields(), R"([[3, "stale"]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK(old_writer->Write(std::move(stale_schema_batch)));
-    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> stale_messages,
-                         old_writer->PrepareCommit(/*wait_compaction=*/false,
-                                                   /*commit_identifier=*/3));
-    ASSERT_EQ(1, stale_messages.size());
-    std::shared_ptr<CommitMessageImpl> stale_message =
-        std::dynamic_pointer_cast<CommitMessageImpl>(stale_messages[0]);
-    ASSERT_TRUE(stale_message);
-    ASSERT_EQ(1, stale_message->GetNewFilesIncrement().NewFiles().size());
-    ASSERT_EQ(0, stale_message->GetNewFilesIncrement().NewFiles()[0]->schema_id);
-    ASSERT_OK(CommitMessages(table_path_, options, stale_messages, /*commit_identifier=*/3));
-
-    ASSERT_OK_AND_ASSIGN(std::optional<Snapshot> snapshot,
-                         LatestSnapshot(table_path_, options, dir_->GetFileSystem()));
-    ASSERT_TRUE(snapshot);
-    ASSERT_EQ(1, snapshot->SchemaId());
-
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows, ReadRows(table_path_, options, pool_));
-    AssertResultEquals(rows.data, EvolvedFields(),
-                       R"([[0, 1, "old", null], [0, 2, "new", 20], [0, 3, "stale", null]])");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest, NonRealtimePkOldWriterCanOverwriteNewColumnWithNull) {
-    std::map<std::string, std::string> options = BaseOptions();
-    ASSERT_OK(
-        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
-                         CreateWriter(table_path_, options));
-
-    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
-                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
-                            DataField(2, EvolvedFields()[2])},
-                           /*highest_field_id=*/2, options));
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
-                         MakeBatch(EvolvedFields(), R"([[1, "new", 20]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> new_messages,
-                         WriteWithNewWriter(table_path_, options, std::move(new_schema_batch),
-                                            /*commit_identifier=*/1));
-    ASSERT_OK(CommitMessages(table_path_, options, new_messages, /*commit_identifier=*/1));
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> old_schema_batch,
-                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK(old_writer->Write(std::move(old_schema_batch)));
-    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> old_messages,
-                         old_writer->PrepareCommit(/*wait_compaction=*/false,
-                                                   /*commit_identifier=*/2));
-    ASSERT_OK(CommitMessages(table_path_, options, old_messages, /*commit_identifier=*/2));
-
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows, ReadRows(table_path_, options, pool_));
-    AssertResultEquals(rows.data, EvolvedFields(), R"([[0, 1, "old", null]])");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest, NonRealtimePkNewWriterIndexesNewSchemaColumn) {
-    std::map<std::string, std::string> options = BaseOptions();
-    ASSERT_OK(
-        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> old_schema_batch,
-                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> old_messages,
-                         WriteWithNewWriter(table_path_, options, std::move(old_schema_batch),
-                                            /*commit_identifier=*/1));
-    ASSERT_OK(CommitMessages(table_path_, options, old_messages, /*commit_identifier=*/1));
-
-    std::map<std::string, std::string> options_v1 = options;
-    options_v1["file-index.bitmap.columns"] = "extra";
-    options_v1[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B";
-    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
-                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
-                            DataField(2, EvolvedFields()[2])},
-                           /*highest_field_id=*/2, options_v1));
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
-                         MakeBatch(EvolvedFields(), R"([[2, "new", 20], [3, "skip", 30]])",
-                                   /*partition=*/{}, /*bucket=*/0));
-    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> new_messages,
-                         WriteWithNewWriter(table_path_, options_v1, std::move(new_schema_batch),
-                                            /*commit_identifier=*/2));
-    ASSERT_EQ(1, new_messages.size());
-    std::shared_ptr<CommitMessageImpl> message =
-        std::dynamic_pointer_cast<CommitMessageImpl>(new_messages[0]);
-    ASSERT_TRUE(message);
-    ASSERT_EQ(1, message->GetNewFilesIncrement().NewFiles().size());
-    std::shared_ptr<DataFileMeta> new_file = message->GetNewFilesIncrement().NewFiles()[0];
-    ASSERT_EQ(1, new_file->schema_id);
-    ASSERT_FALSE(new_file->embedded_index);
-    ASSERT_EQ(1, new_file->extra_files.size());
-    ASSERT_TRUE(new_file->extra_files[0]);
-    std::string index_path =
-        PathUtil::JoinPath(table_path_, "bucket-0/" + new_file->extra_files[0].value());
-    ASSERT_OK_AND_ASSIGN(bool index_exists, dir_->GetFileSystem()->Exists(index_path));
-    ASSERT_TRUE(index_exists);
-
-    ASSERT_OK(CommitMessages(table_path_, options_v1, new_messages, /*commit_identifier=*/2));
-    std::shared_ptr<Predicate> predicate = PredicateBuilder::Equal(
-        /*field_index=*/2, /*field_name=*/"extra", FieldType::INT, Literal(20));
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows,
-                         ReadRows(table_path_, options_v1, pool_, /*realtime_context=*/nullptr,
-                                  predicate, /*enable_predicate_filter=*/true));
-    AssertResultEquals(rows.data, EvolvedFields(), R"([[0, 2, "new", 20]])");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest, NonRealtimePkEmbeddedFileIndexFailsValueScan) {
-    std::map<std::string, std::string> options = BaseOptions();
-    options["file-index.bitmap.columns"] = "payload";
-    options[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1MB";
-    ASSERT_OK(
-        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
-                         MakeBatch(BaseFields(), R"([[1, "a"], [2, "b"]])",
-                                   /*partition=*/{}, /*bucket=*/0));
-    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> messages,
-                         WriteWithNewWriter(table_path_, options, std::move(batch),
-                                            /*commit_identifier=*/1));
-    ASSERT_EQ(1, messages.size());
-    std::shared_ptr<CommitMessageImpl> message =
-        std::dynamic_pointer_cast<CommitMessageImpl>(messages[0]);
-    ASSERT_TRUE(message);
-    ASSERT_EQ(1, message->GetNewFilesIncrement().NewFiles().size());
-    ASSERT_TRUE(message->GetNewFilesIncrement().NewFiles()[0]->embedded_index);
-    ASSERT_OK(CommitMessages(table_path_, options, messages, /*commit_identifier=*/1));
-
-    std::shared_ptr<Predicate> predicate =
-        PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"payload", FieldType::STRING,
-                                Literal(FieldType::STRING, "a", 1));
-    ASSERT_NOK_WITH_MSG(
-        ScanTable(table_path_, options, pool_, /*realtime_context=*/nullptr, predicate),
-        "do not support embedded index in DataFileMeta");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest, NonRealtimePkSortedIndexConfigDoesNotWriteIndexOnDataWrite) {
-    std::map<std::string, std::string> options = BaseOptions();
-    options[Options::PK_BTREE_INDEX_COLUMNS] = "payload";
-    ASSERT_OK(
-        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
-                         MakeBatch(BaseFields(), R"([[1, "a"], [2, "b"]])",
-                                   /*partition=*/{}, /*bucket=*/0));
-    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> messages,
-                         WriteWithNewWriter(table_path_, options, std::move(batch),
-                                            /*commit_identifier=*/1));
-    ASSERT_EQ(1, messages.size());
-    std::shared_ptr<CommitMessageImpl> message =
-        std::dynamic_pointer_cast<CommitMessageImpl>(messages[0]);
-    ASSERT_TRUE(message);
-    ASSERT_TRUE(message->GetNewFilesIncrement().NewIndexFiles().empty());
-    ASSERT_TRUE(message->GetCompactIncrement().NewIndexFiles().empty());
-    ASSERT_OK(CommitMessages(table_path_, options, messages, /*commit_identifier=*/1));
-    ASSERT_OK_AND_ASSIGN(std::optional<Snapshot> snapshot,
-                         LatestSnapshot(table_path_, options, dir_->GetFileSystem()));
-    ASSERT_TRUE(snapshot);
-    ASSERT_FALSE(snapshot->IndexManifest());
-
-    std::shared_ptr<Predicate> predicate =
-        PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"payload", FieldType::STRING,
-                                Literal(FieldType::STRING, "a", 1));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
-                         ScanTable(table_path_, options, pool_,
-                                   /*realtime_context=*/nullptr, predicate));
-    ASSERT_EQ(0, CountIndexedSplits(plan));
-    std::vector<std::shared_ptr<DataFileMeta>> planned_files = DataFilesFromPlan(plan);
-    ASSERT_EQ(1, planned_files.size());
-    ASSERT_EQ(0, planned_files[0]->schema_id);
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows,
-                         ReadRows(table_path_, options, pool_, /*realtime_context=*/nullptr,
-                                  predicate, /*enable_predicate_filter=*/true));
-    AssertResultEquals(rows.data, BaseFields(), R"([[0, 1, "a"]])");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest, RealtimeAppendRejectsDataEvolutionAtWriterCreation) {
-    std::map<std::string, std::string> create_options = BaseOptions();
-    ASSERT_OK(
-        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{}, create_options));
-
-    std::map<std::string, std::string> write_options = BaseOptions();
-    write_options[Options::DATA_EVOLUTION_ENABLED] = "true";
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_NOK_WITH_MSG(CreateWriter(table_path_, write_options, realtime_context),
-                        "real-time append write does not support data evolution");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest, RealtimeAppendScanRejectsDataEvolutionTable) {
-    std::map<std::string, std::string> options = DataEvolutionOptions();
-    arrow::FieldVector fields = DataEvolutionFields();
-    ASSERT_OK(CreateTable(dir_->Str(), arrow::schema(fields), /*primary_keys=*/{}, options));
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
-                         MakeUnbucketedBatch(fields, R"([[1, "old", "base"]])",
-                                             /*partition=*/{}));
-    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> messages,
-                         WriteWithNewWriter(table_path_, options, std::move(batch),
-                                            /*commit_identifier=*/1,
-                                            /*write_schema=*/{"f0", "f1", "f2"}));
-    ASSERT_OK(CommitMessages(table_path_, options, messages, /*commit_identifier=*/1));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create());
-
-    ASSERT_NOK_WITH_MSG(ScanTable(table_path_, options, pool_, realtime_context),
-                        "real-time union read requires fixed bucket mode");
-
-    std::map<std::string, std::string> fixed_bucket_options = options;
-    fixed_bucket_options[Options::BUCKET] = "1";
-    ASSERT_NOK_WITH_MSG(ScanTable(table_path_, fixed_bucket_options, pool_, realtime_context),
-                        "real-time union read does not support data evolution");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest, RealtimeAppendReuseContextKeepsOldMemorySchemaAfterAlter) {
-    std::map<std::string, std::string> options = BaseOptions();
-    ASSERT_OK(CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{}, options));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
-                         CreateWriter(table_path_, options, realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> base_batch,
-                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK(old_writer->Write(std::move(base_batch)));
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> base_progress,
-                         old_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
-    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id,
-                         CommitRealtimeMessages(table_path_, options, base_progress,
-                                                /*commit_identifier=*/1));
-    ASSERT_OK(old_writer->RefreshCommittedSnapshot(snapshot_id));
-
-    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
-                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
-                            DataField(2, EvolvedFields()[2])},
-                           /*highest_field_id=*/2, options));
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> reused_context_writer,
-                         CreateWriter(table_path_, options, realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
-                         MakeBatch(EvolvedFields(), R"([[2, "new", 20]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_NOK_WITH_MSG(reused_context_writer->Write(std::move(new_schema_batch)),
-                        "ArrowArray struct has 3 children, expected 2");
-
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows,
-                         ReadRows(table_path_, options, pool_, realtime_context));
-    AssertResultEquals(rows.data, EvolvedFields(), R"([[0, 1, "old", null]])");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest, RealtimeAppendOldWriterAfterAlterCommitsOldSchemaFile) {
-    std::map<std::string, std::string> options = BaseOptions();
-    ASSERT_OK(CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{}, options));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
-                         CreateWriter(table_path_, options, realtime_context));
-
-    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
-                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
-                            DataField(2, EvolvedFields()[2])},
-                           /*highest_field_id=*/2, options));
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> old_schema_batch,
-                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK(old_writer->Write(std::move(old_schema_batch)));
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult memory_rows,
-                         ReadRows(table_path_, options, pool_, realtime_context));
-    AssertResultEquals(memory_rows.data, EvolvedFields(), R"([[0, 1, "old", null]])");
-
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
-                         old_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
-    ASSERT_EQ(1, progress.size());
-    std::shared_ptr<CommitMessageImpl> message =
-        std::dynamic_pointer_cast<CommitMessageImpl>(progress[0].commit_message);
-    ASSERT_TRUE(message);
-    ASSERT_EQ(1, message->GetNewFilesIncrement().NewFiles().size());
-    ASSERT_EQ(0, message->GetNewFilesIncrement().NewFiles()[0]->schema_id);
-
-    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, CommitRealtimeMessages(table_path_, options, progress,
-                                                                     /*commit_identifier=*/1));
-    ASSERT_OK(old_writer->RefreshCommittedSnapshot(snapshot_id));
-    ASSERT_OK_AND_ASSIGN(std::optional<Snapshot> snapshot,
-                         LatestSnapshot(table_path_, options, dir_->GetFileSystem()));
-    ASSERT_TRUE(snapshot);
-    ASSERT_EQ(1, snapshot->SchemaId());
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult disk_rows, ReadRows(table_path_, options, pool_));
-    AssertResultEquals(disk_rows.data, EvolvedFields(), R"([[0, 1, "old", null]])");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest, RealtimeAppendNewContextUsesNewSchemaAfterAlter) {
-    std::map<std::string, std::string> options = BaseOptions();
-    ASSERT_OK(CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{}, options));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> old_realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
-                         CreateWriter(table_path_, options, old_realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> base_batch,
-                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK(old_writer->Write(std::move(base_batch)));
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> base_progress,
-                         old_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
-    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id,
-                         CommitRealtimeMessages(table_path_, options, base_progress,
-                                                /*commit_identifier=*/1));
-    ASSERT_OK(old_writer->RefreshCommittedSnapshot(snapshot_id));
-
-    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
-                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
-                            DataField(2, EvolvedFields()[2])},
-                           /*highest_field_id=*/2, options));
-
-    std::map<std::string, std::string> options_v1 = options;
-    options_v1["file-index.bitmap.columns"] = "extra";
-    options_v1[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B";
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> new_realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> new_writer,
-                         CreateWriter(table_path_, options_v1, new_realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
-                         MakeBatch(EvolvedFields(), R"([[2, "new", 20]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK(new_writer->Write(std::move(new_schema_batch)));
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult memory_rows,
-                         ReadRows(table_path_, options_v1, pool_, new_realtime_context));
-    AssertResultEquals(memory_rows.data, EvolvedFields(),
-                       R"([[0, 1, "old", null], [0, 2, "new", 20]])");
-
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> new_progress,
-                         new_writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
-    ASSERT_EQ(1, new_progress.size());
-    std::shared_ptr<CommitMessageImpl> message =
-        std::dynamic_pointer_cast<CommitMessageImpl>(new_progress[0].commit_message);
-    ASSERT_TRUE(message);
-    ASSERT_EQ(1, message->GetNewFilesIncrement().NewFiles().size());
-    std::shared_ptr<DataFileMeta> new_file = message->GetNewFilesIncrement().NewFiles()[0];
-    ASSERT_EQ(1, new_file->schema_id);
-    ASSERT_FALSE(new_file->embedded_index);
-    ASSERT_EQ(1, new_file->extra_files.size());
-    ASSERT_TRUE(new_file->extra_files[0]);
-    std::string index_path =
-        PathUtil::JoinPath(table_path_, "bucket-0/" + new_file->extra_files[0].value());
-    ASSERT_OK_AND_ASSIGN(bool index_exists, dir_->GetFileSystem()->Exists(index_path));
-    ASSERT_TRUE(index_exists);
-
-    ASSERT_OK_AND_ASSIGN(snapshot_id, CommitRealtimeMessages(table_path_, options_v1, new_progress,
-                                                             /*commit_identifier=*/2));
-    ASSERT_OK(new_writer->RefreshCommittedSnapshot(snapshot_id));
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows,
-                         ReadRows(table_path_, options_v1, pool_, new_realtime_context));
-    AssertResultEquals(rows.data, EvolvedFields(), R"([[0, 1, "old", null], [0, 2, "new", 20]])");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest, RealtimePkRejectsDataEvolutionAtWriterCreation) {
-    std::map<std::string, std::string> create_options = BaseOptions();
-    ASSERT_OK(CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"},
-                          create_options));
-
-    std::map<std::string, std::string> write_options = BaseOptions();
-    write_options[Options::DATA_EVOLUTION_ENABLED] = "true";
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_NOK_WITH_MSG(CreateWriter(table_path_, write_options, realtime_context),
-                        "PK realtime v1 does not support data evolution");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest, RealtimePkReuseContextKeepsOldMemorySchemaAfterAlter) {
-    std::map<std::string, std::string> options = BaseOptions();
-    ASSERT_OK(
-        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
-                         CreateWriter(table_path_, options, realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> base_batch,
-                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK(old_writer->Write(std::move(base_batch)));
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> base_progress,
-                         old_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
-    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id,
-                         CommitRealtimeMessages(table_path_, options, base_progress,
-                                                /*commit_identifier=*/1));
-    ASSERT_OK(old_writer->RefreshCommittedSnapshot(snapshot_id));
-
-    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
-                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
-                            DataField(2, EvolvedFields()[2])},
-                           /*highest_field_id=*/2, options));
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> reused_context_writer,
-                         CreateWriter(table_path_, options, realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
-                         MakeBatch(EvolvedFields(), R"([[2, "new", 20]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_NOK_WITH_MSG(reused_context_writer->Write(std::move(new_schema_batch)),
-                        "ArrowArray struct has 3 children, expected 2");
-
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows,
-                         ReadRows(table_path_, options, pool_, realtime_context));
-    AssertResultEquals(rows.data, EvolvedFields(), R"([[0, 1, "old", null]])");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest, RealtimePkOldWriterAfterAlterReadsNewColumnAsNull) {
-    std::map<std::string, std::string> options = BaseOptions();
-    ASSERT_OK(
-        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
-                         CreateWriter(table_path_, options, realtime_context));
-
-    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
-                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
-                            DataField(2, EvolvedFields()[2])},
-                           /*highest_field_id=*/2, options));
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> old_schema_batch,
-                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK(old_writer->Write(std::move(old_schema_batch)));
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult memory_rows,
-                         ReadRows(table_path_, options, pool_, realtime_context));
-    AssertResultEquals(memory_rows.data, EvolvedFields(), R"([[0, 1, "old", null]])");
-
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
-                         old_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
-    ASSERT_EQ(1, progress.size());
-    std::shared_ptr<CommitMessageImpl> message =
-        std::dynamic_pointer_cast<CommitMessageImpl>(progress[0].commit_message);
-    ASSERT_TRUE(message);
-    ASSERT_EQ(1, message->GetNewFilesIncrement().NewFiles().size());
-    ASSERT_EQ(0, message->GetNewFilesIncrement().NewFiles()[0]->schema_id);
-
-    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, CommitRealtimeMessages(table_path_, options, progress,
-                                                                     /*commit_identifier=*/1));
-    ASSERT_OK(old_writer->RefreshCommittedSnapshot(snapshot_id));
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult disk_rows, ReadRows(table_path_, options, pool_));
-    AssertResultEquals(disk_rows.data, EvolvedFields(), R"([[0, 1, "old", null]])");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest,
-       RealtimePkOldWriterAfterAlterCannotCommitBehindNewContextOffset) {
-    std::map<std::string, std::string> options = BaseOptions();
-    ASSERT_OK(
-        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
-                         CreateWriter(table_path_, options, realtime_context));
-
-    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
-                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
-                            DataField(2, EvolvedFields()[2])},
-                           /*highest_field_id=*/2, options));
-
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> new_realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> new_writer,
-                         CreateWriter(table_path_, options, new_realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
-                         MakeBatch(EvolvedFields(), R"([[1, "new", 20]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK(new_writer->Write(std::move(new_schema_batch)));
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> new_progress,
-                         new_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
-    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id,
-                         CommitRealtimeMessages(table_path_, options, new_progress,
-                                                /*commit_identifier=*/1));
-    ASSERT_OK(new_writer->RefreshCommittedSnapshot(snapshot_id));
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> old_schema_batch,
-                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK(old_writer->Write(std::move(old_schema_batch)));
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult realtime_rows,
-                         ReadRows(table_path_, options, pool_, realtime_context));
-    AssertResultEquals(realtime_rows.data, EvolvedFields(), R"([[0, 1, "new", 20]])");
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> old_progress,
-                         old_writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
-    ASSERT_NOK_WITH_MSG(CommitRealtimeMessages(table_path_, options, old_progress,
-                                               /*commit_identifier=*/2),
-                        "real-time commit offsets for bucket 0 are not contiguous");
-
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult disk_rows, ReadRows(table_path_, options, pool_));
-    AssertResultEquals(disk_rows.data, EvolvedFields(), R"([[0, 1, "new", 20]])");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest, RealtimePkNewContextUsesNewSchemaAfterAlter) {
-    std::map<std::string, std::string> options = BaseOptions();
-    ASSERT_OK(
-        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> old_realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
-                         CreateWriter(table_path_, options, old_realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> base_batch,
-                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK(old_writer->Write(std::move(base_batch)));
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> base_progress,
-                         old_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
-    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id,
-                         CommitRealtimeMessages(table_path_, options, base_progress,
-                                                /*commit_identifier=*/1));
-    ASSERT_OK(old_writer->RefreshCommittedSnapshot(snapshot_id));
-
-    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
-                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
-                            DataField(2, EvolvedFields()[2])},
-                           /*highest_field_id=*/2, options));
-
-    std::map<std::string, std::string> options_v1 = options;
-    options_v1["file-index.bitmap.columns"] = "extra";
-    options_v1[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B";
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> new_realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> new_writer,
-                         CreateWriter(table_path_, options_v1, new_realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
-                         MakeBatch(EvolvedFields(), R"([[2, "new", 20]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK(new_writer->Write(std::move(new_schema_batch)));
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult memory_rows,
-                         ReadRows(table_path_, options_v1, pool_, new_realtime_context));
-    AssertResultEquals(memory_rows.data, EvolvedFields(),
-                       R"([[0, 1, "old", null], [0, 2, "new", 20]])");
-
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> new_progress,
-                         new_writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
-    ASSERT_EQ(1, new_progress.size());
-    std::shared_ptr<CommitMessageImpl> message =
-        std::dynamic_pointer_cast<CommitMessageImpl>(new_progress[0].commit_message);
-    ASSERT_TRUE(message);
-    ASSERT_EQ(1, message->GetNewFilesIncrement().NewFiles().size());
-    std::shared_ptr<DataFileMeta> new_file = message->GetNewFilesIncrement().NewFiles()[0];
-    ASSERT_EQ(1, new_file->schema_id);
-    ASSERT_FALSE(new_file->embedded_index);
-    ASSERT_EQ(1, new_file->extra_files.size());
-    ASSERT_TRUE(new_file->extra_files[0]);
-    std::string index_path =
-        PathUtil::JoinPath(table_path_, "bucket-0/" + new_file->extra_files[0].value());
-    ASSERT_OK_AND_ASSIGN(bool index_exists, dir_->GetFileSystem()->Exists(index_path));
-    ASSERT_TRUE(index_exists);
-
-    ASSERT_OK_AND_ASSIGN(snapshot_id, CommitRealtimeMessages(table_path_, options_v1, new_progress,
-                                                             /*commit_identifier=*/2));
-    ASSERT_OK(new_writer->RefreshCommittedSnapshot(snapshot_id));
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows,
-                         ReadRows(table_path_, options_v1, pool_, new_realtime_context));
-    AssertResultEquals(rows.data, EvolvedFields(), R"([[0, 1, "old", null], [0, 2, "new", 20]])");
-}
-
-}  // namespace
-}  // namespace paimon::test

From 2f7d228ab5d1521486e13c2bfed7afaeb9bafc9c Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Mon, 24 Aug 2026 11:21:45 +0800
Subject: [PATCH 13/62] refactor(mergetree): accept sorted key-value readers

---
 .../core/mergetree/merge_tree_writer.cpp      |  95 ++++---
 src/paimon/core/mergetree/merge_tree_writer.h |   3 +
 .../core/mergetree/merge_tree_writer_test.cpp | 236 ++++++++++++++++++
 3 files changed, 293 insertions(+), 41 deletions(-)

diff --git a/src/paimon/core/mergetree/merge_tree_writer.cpp b/src/paimon/core/mergetree/merge_tree_writer.cpp
index 3b6806c73..49961536a 100644
--- a/src/paimon/core/mergetree/merge_tree_writer.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer.cpp
@@ -154,6 +154,59 @@ Status MergeTreeWriter::Write(std::unique_ptr<RecordBatch>&& moved_batch) {
     return Status::OK();
 }
 
+Status MergeTreeWriter::WriteSortedReaders(
+    std::vector<std::unique_ptr<KeyValueRecordReader>>&& readers) {
+    auto raw_readers_guard = ScopeGuard([&]() -> void {
+        for (std::unique_ptr<KeyValueRecordReader>& reader : readers) {
+            if (reader != nullptr) {
+                reader->Close();
+            }
+        }
+    });
+    if (readers.empty()) {
+        return Status::Invalid("sorted readers must not be empty");
+    }
+    for (const std::unique_ptr<KeyValueRecordReader>& reader : readers) {
+        if (reader == nullptr) {
+            return Status::Invalid("sorted readers must not contain null reader");
+        }
+    }
+
+    auto sort_merge_reader = std::make_unique<SortMergeReaderWithLoserTree>(
+        std::move(readers), key_comparator_, user_defined_seq_comparator_, merge_function_wrapper_);
+    auto create_consumer = [target_schema = write_schema_, pool = pool_]()
+        -> Result<std::unique_ptr<RowToArrowArrayConverter<KeyValue, KeyValueBatch>>> {
+        return KeyValueMetaProjectionConsumer::Create(target_schema, pool);
+    };
+    auto async_key_value_producer_consumer =
+        std::make_unique<AsyncKeyValueProducerAndConsumer<KeyValue, KeyValueBatch>>(
+            std::move(sort_merge_reader), create_consumer, options_.GetWriteBatchSize(), 1, pool_);
+    raw_readers_guard.Release();
+    ScopeGuard async_readers_guard([&]() -> void { async_key_value_producer_consumer->Close(); });
+    std::unique_ptr<RollingFileWriter<KeyValueBatch, std::shared_ptr<DataFileMeta>>> rolling_writer;
+    PAIMON_ASSIGN_OR_RAISE(rolling_writer, CreateRollingRowWriter());
+    ScopeGuard abort_writer_guard([&]() -> void { rolling_writer->Abort(); });
+    while (true) {
+        PAIMON_ASSIGN_OR_RAISE(KeyValueBatch key_value_batch,
+                               async_key_value_producer_consumer->NextBatch());
+        if (key_value_batch.batch == nullptr) {
+            break;
+        }
+        PAIMON_RETURN_NOT_OK(rolling_writer->Write(std::move(key_value_batch)));
+    }
+    PAIMON_RETURN_NOT_OK(rolling_writer->Close());
+    PAIMON_ASSIGN_OR_RAISE(std::vector<std::shared_ptr<DataFileMeta>> flushed_files,
+                           rolling_writer->GetResult());
+    abort_writer_guard.Release();
+
+    for (const std::shared_ptr<DataFileMeta>& flushed_file : flushed_files) {
+        new_files_.emplace_back(flushed_file);
+        PAIMON_RETURN_NOT_OK(compact_manager_->AddNewFile(flushed_file));
+    }
+    metrics_->Merge(rolling_writer->GetMetrics());
+    return Status::OK();
+}
+
 Status MergeTreeWriter::Compact(bool full_compaction) {
     return FlushWriteBuffer(/*wait_for_latest_compaction=*/true, full_compaction);
 }
@@ -256,49 +309,9 @@ Status MergeTreeWriter::FlushWriteBuffer(bool wait_for_latest_compaction,
             wait_for_latest_compaction = true;
         }
         auto cleanup_guard = ScopeGuard([&]() { write_buffer_->Clear(); });
-        // 1. flush write buffer to get sorted readers
         PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
                                write_buffer_->CreateReaders());
-        // 2. prepare loser tree sort merge reader
-        auto sort_merge_reader = std::make_unique<SortMergeReaderWithLoserTree>(
-            std::move(readers), key_comparator_, user_defined_seq_comparator_,
-            merge_function_wrapper_);
-        // 3. project key value to arrow array
-        auto create_consumer = [target_schema = write_schema_, pool = pool_]()
-            -> Result<std::unique_ptr<RowToArrowArrayConverter<KeyValue, KeyValueBatch>>> {
-            return KeyValueMetaProjectionConsumer::Create(target_schema, pool);
-        };
-        // consumer batch size is WriteBatchSize
-        auto async_key_value_producer_consumer =
-            std::make_unique<AsyncKeyValueProducerAndConsumer<KeyValue, KeyValueBatch>>(
-                std::move(sort_merge_reader), create_consumer, options_.GetWriteBatchSize(),
-                /*projection_thread_num=*/1, pool_);
-        std::unique_ptr<RollingFileWriter<KeyValueBatch, std::shared_ptr<DataFileMeta>>>
-            rolling_writer;
-        PAIMON_ASSIGN_OR_RAISE(rolling_writer, CreateRollingRowWriter());
-        ScopeGuard write_guard([&]() -> void {
-            rolling_writer->Abort();
-            async_key_value_producer_consumer->Close();
-        });
-        while (true) {
-            PAIMON_ASSIGN_OR_RAISE(KeyValueBatch key_value_batch,
-                                   async_key_value_producer_consumer->NextBatch());
-            if (key_value_batch.batch == nullptr) {
-                break;
-            }
-            PAIMON_RETURN_NOT_OK(rolling_writer->Write(std::move(key_value_batch)));
-        }
-        PAIMON_RETURN_NOT_OK(rolling_writer->Close());
-        PAIMON_ASSIGN_OR_RAISE(std::vector<std::shared_ptr<DataFileMeta>> flushed_files,
-                               rolling_writer->GetResult());
-        async_key_value_producer_consumer->Close();
-        write_guard.Release();
-
-        for (const auto& flushed_file : flushed_files) {
-            new_files_.emplace_back(flushed_file);
-            PAIMON_RETURN_NOT_OK(compact_manager_->AddNewFile(flushed_file));
-        }
-        metrics_->Merge(rolling_writer->GetMetrics());
+        PAIMON_RETURN_NOT_OK(WriteSortedReaders(std::move(readers)));
     }
     PAIMON_RETURN_NOT_OK(TrySyncLatestCompaction(wait_for_latest_compaction));
     PAIMON_RETURN_NOT_OK(compact_manager_->TriggerCompaction(forced_full_compaction));
diff --git a/src/paimon/core/mergetree/merge_tree_writer.h b/src/paimon/core/mergetree/merge_tree_writer.h
index febce2afb..542affd81 100644
--- a/src/paimon/core/mergetree/merge_tree_writer.h
+++ b/src/paimon/core/mergetree/merge_tree_writer.h
@@ -51,6 +51,7 @@ class IOManager;
 class FieldsComparator;
 class MemoryPool;
 class Metrics;
+class KeyValueRecordReader;
 template <typename T>
 class MergeFunctionWrapper;
 
@@ -69,6 +70,8 @@ class MergeTreeWriter : public BatchWriter {
 
     Status Write(std::unique_ptr<RecordBatch>&& batch) override;
 
+    Status WriteSortedReaders(std::vector<std::unique_ptr<KeyValueRecordReader>>&& readers);
+
     Status Compact(bool full_compaction) override;
 
     Result<bool> CompactNotCompleted() override;
diff --git a/src/paimon/core/mergetree/merge_tree_writer_test.cpp b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
index 2155647a1..675ce3198 100644
--- a/src/paimon/core/mergetree/merge_tree_writer_test.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
@@ -24,6 +24,7 @@
 #include <map>
 #include <optional>
 #include <utility>
+#include <vector>
 
 #include "arrow/api.h"
 #include "arrow/array/array_base.h"
@@ -42,6 +43,7 @@
 #include "paimon/core/io/compact_increment.h"
 #include "paimon/core/io/data_file_path_factory.h"
 #include "paimon/core/io/data_increment.h"
+#include "paimon/core/io/key_value_record_reader.h"
 #include "paimon/core/manifest/file_source.h"
 #include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
 #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
@@ -52,6 +54,8 @@
 #include "paimon/fs/file_system.h"
 #include "paimon/fs/local/local_file_system.h"
 #include "paimon/memory/memory_pool.h"
+#include "paimon/testing/mock/mock_file_batch_reader.h"
+#include "paimon/testing/mock/mock_key_value_data_file_record_reader.h"
 #include "paimon/testing/utils/binary_row_generator.h"
 #include "paimon/testing/utils/io_exception_helper.h"
 #include "paimon/testing/utils/read_result_collector.h"
@@ -64,6 +68,60 @@ class MergeFunctionWrapper;
 }  // namespace paimon
 
 namespace paimon::test {
+namespace {
+
+class TrackingKeyValueRecordReader : public KeyValueRecordReader {
+ public:
+    TrackingKeyValueRecordReader(std::unique_ptr<KeyValueRecordReader>&& inner_reader,
+                                 bool* closed_flag)
+        : inner_reader_(std::move(inner_reader)), closed_flag_(closed_flag) {}
+
+    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatch() override {
+        return inner_reader_->NextBatch();
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return inner_reader_->GetReaderMetrics();
+    }
+
+    void Close() override {
+        if (closed_flag_ != nullptr) {
+            *closed_flag_ = true;
+        }
+        inner_reader_->Close();
+    }
+
+ private:
+    std::unique_ptr<KeyValueRecordReader> inner_reader_;
+    bool* closed_flag_;
+};
+
+class ErrorKeyValueRecordReader : public KeyValueRecordReader {
+ public:
+    ErrorKeyValueRecordReader(Status status, bool* closed_flag)
+        : status_(std::move(status)), closed_flag_(closed_flag) {}
+
+    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatch() override {
+        return status_;
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return nullptr;
+    }
+
+    void Close() override {
+        if (closed_flag_ != nullptr) {
+            *closed_flag_ = true;
+        }
+    }
+
+ private:
+    Status status_;
+    bool* closed_flag_;
+};
+
+}
+
 class MergeTreeWriterTest : public ::testing::TestWithParam<bool> {
  public:
     class FakeCompactManager : public paimon::CompactManager {
@@ -211,6 +269,21 @@ class MergeTreeWriterTest : public ::testing::TestWithParam<bool> {
             writer_compact_manager, io_manager, /*enable_multi_thread_spill=*/false, pool_);
     }
 
+    std::unique_ptr<KeyValueRecordReader> CreateSingleReader(
+        const std::shared_ptr<arrow::Array>& array, int32_t batch_size = 16) const {
+        std::vector<DataField> write_fields = {SpecialFields::SequenceNumber(),
+                                               SpecialFields::ValueKind()};
+        write_fields.insert(write_fields.end(), value_fields_.begin(), value_fields_.end());
+        std::shared_ptr<arrow::Schema> write_schema =
+            DataField::ConvertDataFieldsToArrowSchema(write_fields);
+        std::shared_ptr<arrow::Schema> key_schema =
+            arrow::schema(arrow::FieldVector({write_schema->field(2)}));
+        auto file_batch_reader =
+            std::make_unique<MockFileBatchReader>(array, array->type(), batch_size);
+        return std::make_unique<MockKeyValueDataFileRecordReader>(
+            std::move(file_batch_reader), key_schema, value_schema_, 0, pool_);
+    }
+
  private:
     std::shared_ptr<MemoryPool> pool_;
     std::shared_ptr<FileSystem> file_system_;
@@ -293,6 +366,29 @@ TEST_P(MergeTreeWriterTest, TestSimple) {
     DataIncrement expected_data_increment({expected_data_file_meta}, /*deleted_files=*/{},
                                           /*changelog_files=*/{});
     ASSERT_EQ(expected_data_increment, commit_increment.GetNewFilesIncrement());
+
+    auto sorted_reader_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(write_type_, R"([
+      [2, 0, "Alice", 10, 0, 13.1],
+      [0, 0, "Lucy", 20, 1, 14.1],
+      [1, 0, "Paul", 20, 1, null]
+    ])")
+            .ValueOrDie());
+    auto sorted_reader_path_factory = std::make_shared<DataFilePathFactory>();
+    ASSERT_OK(sorted_reader_path_factory->Init(dir->Str() + "/sorted-readers", "orc",
+                                               options.DataFilePrefix(), nullptr));
+    ASSERT_OK_AND_ASSIGN(auto sorted_reader_writer,
+                         CreateMergeWriter(-1, dir->Str(), sorted_reader_path_factory, 1, options));
+    std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
+    sorted_readers.push_back(CreateSingleReader(sorted_reader_array));
+    ASSERT_OK(sorted_reader_writer->WriteSortedReaders(std::move(sorted_readers)));
+    ASSERT_OK_AND_ASSIGN(CommitIncrement sorted_reader_commit_increment,
+                         sorted_reader_writer->PrepareCommit(false));
+    ASSERT_OK(sorted_reader_writer->Close());
+    ASSERT_EQ(1, sorted_reader_commit_increment.GetNewFilesIncrement().NewFiles().size());
+    std::string sorted_reader_path = sorted_reader_path_factory->ToPath(
+        sorted_reader_commit_increment.GetNewFilesIncrement().NewFiles()[0]);
+    CheckFileContent(sorted_reader_path, expected_array);
 }
 
 TEST_P(MergeTreeWriterTest, TestWriteMultiBatch) {
@@ -374,6 +470,146 @@ TEST_P(MergeTreeWriterTest, TestWriteMultiBatch) {
     DataIncrement expected_data_increment({expected_data_file_meta}, /*deleted_files=*/{},
                                           /*changelog_files=*/{});
     ASSERT_EQ(expected_data_increment, commit_increment.GetNewFilesIncrement());
+
+    auto sorted_reader_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(write_type_, R"([
+      [16, 0, "Alice", 10, 0, 113.1],
+      [14, 0, "Lucy", 20, 1, 114.1],
+      [13, 0, "Paul", 20, 1, 15.1],
+      [15, 0, "Skye", 10, 0, 118.1]
+    ])")
+            .ValueOrDie());
+    auto sorted_reader_path_factory = std::make_shared<DataFilePathFactory>();
+    ASSERT_OK(sorted_reader_path_factory->Init(dir->Str() + "/sorted-readers", "orc",
+                                               options.DataFilePrefix(), nullptr));
+    ASSERT_OK_AND_ASSIGN(auto sorted_reader_writer,
+                         CreateMergeWriter(9, dir->Str(), sorted_reader_path_factory, 0, options));
+    std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
+    sorted_readers.push_back(CreateSingleReader(sorted_reader_array));
+    ASSERT_OK(sorted_reader_writer->WriteSortedReaders(std::move(sorted_readers)));
+    ASSERT_OK_AND_ASSIGN(CommitIncrement sorted_reader_commit_increment,
+                         sorted_reader_writer->PrepareCommit(false));
+    ASSERT_OK(sorted_reader_writer->Close());
+    ASSERT_EQ(1, sorted_reader_commit_increment.GetNewFilesIncrement().NewFiles().size());
+    std::string sorted_reader_path = sorted_reader_path_factory->ToPath(
+        sorted_reader_commit_increment.GetNewFilesIncrement().NewFiles()[0]);
+    CheckFileContent(sorted_reader_path, expected_array);
+}
+
+TEST_P(MergeTreeWriterTest, TestSortedReaders) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options,
+                         CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}}));
+
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    auto path_factory = std::make_shared<DataFilePathFactory>();
+    ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr));
+    std::string uuid = path_factory->uuid_;
+
+    ASSERT_OK_AND_ASSIGN(auto merge_writer,
+                         CreateMergeWriter(4, dir->Str(), path_factory, 7, options));
+
+    auto sorted_reader_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(write_type_, R"([
+      [7, 0, "Alice", 20, 1, 17.1],
+      [9, 0, "Lucy", 30, 2, 19.1],
+      [8, 3, "Paul", 10, 3, null]
+    ])")
+            .ValueOrDie());
+
+    std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
+    sorted_readers.push_back(CreateSingleReader(sorted_reader_array));
+
+    ASSERT_OK(merge_writer->WriteSortedReaders(std::move(sorted_readers)));
+    ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment, merge_writer->PrepareCommit(false));
+    ASSERT_OK(merge_writer->Close());
+
+    std::string expected_data_file_name = "data-" + uuid + "-0.orc";
+    std::string expected_data_file_path = dir->Str() + "/" + expected_data_file_name;
+    ASSERT_OK_AND_ASSIGN(FileStatus data_file_status,
+                         options.GetFileSystem()->GetFileStatus(expected_data_file_path));
+
+    std::shared_ptr<arrow::ChunkedArray> expected_array;
+    ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(write_type_, {R"([
+      [7, 0, "Alice", 20, 1, 17.1],
+      [9, 0, "Lucy", 30, 2, 19.1],
+      [8, 3, "Paul", 10, 3, null]
+    ])"},
+                                                                 &expected_array)
+                    .ok());
+    CheckFileContent(expected_data_file_path, expected_array);
+
+    ASSERT_TRUE(commit_increment.GetCompactIncrement().IsEmpty());
+    ASSERT_EQ(1, commit_increment.GetNewFilesIncrement().NewFiles().size());
+    const std::shared_ptr<DataFileMeta>& new_file =
+        commit_increment.GetNewFilesIncrement().NewFiles()[0];
+    ASSERT_EQ(expected_data_file_name, new_file->file_name);
+    ASSERT_EQ(data_file_status.GetLen(), new_file->file_size);
+    ASSERT_EQ(3, new_file->row_count);
+    ASSERT_EQ(7, new_file->min_sequence_number);
+    ASSERT_EQ(9, new_file->max_sequence_number);
+    ASSERT_EQ(7, new_file->schema_id);
+    ASSERT_EQ(1, new_file->delete_row_count);
+}
+
+TEST_P(MergeTreeWriterTest, TestSortedReaderOwnership) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options,
+                         CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}}));
+
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    auto path_factory = std::make_shared<DataFilePathFactory>();
+    ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr));
+
+    ASSERT_OK_AND_ASSIGN(auto merge_writer,
+                         CreateMergeWriter(-1, dir->Str(), path_factory, 0, options));
+
+    auto sorted_reader_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(write_type_, R"([
+      [0, 0, "Alice", 10, 0, 13.1]
+    ])")
+            .ValueOrDie());
+
+    bool closed = false;
+    std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
+    sorted_readers.push_back(std::make_unique<TrackingKeyValueRecordReader>(
+        CreateSingleReader(sorted_reader_array), &closed));
+
+    ASSERT_OK(merge_writer->WriteSortedReaders(std::move(sorted_readers)));
+    ASSERT_TRUE(closed);
+    ASSERT_OK(merge_writer->Close());
+}
+
+TEST_P(MergeTreeWriterTest, TestSortedReaderFailure) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options,
+                         CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}}));
+
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    auto path_factory = std::make_shared<DataFilePathFactory>();
+    ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr));
+
+    ASSERT_OK_AND_ASSIGN(auto merge_writer,
+                         CreateMergeWriter(-1, dir->Str(), path_factory, 0, options));
+
+    std::vector<std::unique_ptr<KeyValueRecordReader>> empty_readers;
+    Status empty_status = merge_writer->WriteSortedReaders(std::move(empty_readers));
+    ASSERT_TRUE(empty_status.IsInvalid());
+
+    std::vector<std::unique_ptr<KeyValueRecordReader>> null_readers;
+    null_readers.push_back(nullptr);
+    Status null_status = merge_writer->WriteSortedReaders(std::move(null_readers));
+    ASSERT_TRUE(null_status.IsInvalid());
+
+    bool failing_reader_closed = false;
+    auto failing_reader = std::make_unique<ErrorKeyValueRecordReader>(
+        Status::IOError("sorted reader failure"), &failing_reader_closed);
+    std::vector<std::unique_ptr<KeyValueRecordReader>> failing_readers;
+    failing_readers.push_back(std::move(failing_reader));
+    Status failing_status = merge_writer->WriteSortedReaders(std::move(failing_readers));
+    ASSERT_TRUE(failing_status.IsIOError());
+    ASSERT_TRUE(failing_reader_closed);
+    ASSERT_OK(merge_writer->Close());
 }
 
 TEST_P(MergeTreeWriterTest, TestSharedShreddingMapDataFileMetaInfo) {

From 3df2037efac097e0069156df0e793cb70e68897e Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Mon, 24 Aug 2026 11:22:00 +0800
Subject: [PATCH 14/62] feat(realtime): adapt prepared primary-key batches

---
 src/paimon/CMakeLists.txt                     |   1 +
 .../merged_key_value_record_reader_test.cpp   | 398 ++++++++++++
 .../core/io/prepared_key_value_reader.cpp     | 565 ++++++++++++++++++
 .../core/io/prepared_key_value_reader.h       |  41 ++
 src/paimon/core/realtime/realtime_fields.h    |  37 ++
 .../core/schema/schema_validation_test.cpp    |   7 +
 6 files changed, 1049 insertions(+)
 create mode 100644 src/paimon/core/io/prepared_key_value_reader.cpp
 create mode 100644 src/paimon/core/io/prepared_key_value_reader.h
 create mode 100644 src/paimon/core/realtime/realtime_fields.h

diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index d2c0a2b4f..62af55ec4 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -286,6 +286,7 @@ set(PAIMON_CORE_SRCS
     core/io/vector_file_batch_reader.cpp
     core/io/file_index_evaluator.cpp
     core/io/key_value_data_file_record_reader.cpp
+    core/io/prepared_key_value_reader.cpp
     core/io/key_value_data_file_writer_factory.cpp
     core/io/key_value_data_file_writer.cpp
     core/io/key_value_in_memory_record_reader.cpp
diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index 1b6b71c69..39714fa29 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -18,8 +18,13 @@
 
 #include "paimon/core/io/merged_key_value_record_reader.h"
 
+#include <cstdint>
+#include <limits>
 #include <memory>
+#include <optional>
+#include <string>
 #include <utility>
+#include <vector>
 
 #include "arrow/api.h"
 #include "arrow/array/array_nested.h"
@@ -27,10 +32,14 @@
 #include "gtest/gtest.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
+#include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/fields_comparator.h"
+#include "paimon/core/io/prepared_key_value_reader.h"
 #include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
 #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
+#include "paimon/core/realtime/realtime_fields.h"
 #include "paimon/memory/memory_pool.h"
+#include "paimon/realtime/offset_range.h"
 #include "paimon/testing/mock/mock_file_batch_reader.h"
 #include "paimon/testing/mock/mock_key_value_data_file_record_reader.h"
 #include "paimon/testing/utils/key_value_checker.h"
@@ -38,6 +47,56 @@
 #include "paimon/testing/utils/testharness.h"
 
 namespace paimon::test {
+
+namespace {
+
+std::shared_ptr<arrow::Field> MakeField(const std::string& name,
+                                        const std::shared_ptr<arrow::DataType>& type,
+                                        int32_t field_id, bool nullable = true) {
+    return DataField::ConvertDataFieldToArrowField(
+        DataField(field_id, arrow::field(name, type, nullable)));
+}
+
+std::shared_ptr<arrow::Schema> MakePreparedSchema(const arrow::FieldVector& value_fields) {
+    arrow::FieldVector prepared_fields = {
+        DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
+        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
+            ->WithNullable(false),
+        DataField::ConvertDataFieldToArrowField(RealtimeOffsetField())->WithNullable(false)};
+    prepared_fields.insert(prepared_fields.end(), value_fields.begin(), value_fields.end());
+    return arrow::schema(prepared_fields);
+}
+
+class TrackingBatchReader : public BatchReader {
+ public:
+    TrackingBatchReader(std::unique_ptr<BatchReader>&& delegate, int32_t* close_count)
+        : delegate_(std::move(delegate)), close_count_(close_count) {}
+
+    Result<ReadBatch> NextBatch() override {
+        return delegate_->NextBatch();
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return delegate_->GetReaderMetrics();
+    }
+
+    void Close() override {
+        if (closed_) {
+            return;
+        }
+        closed_ = true;
+        ++(*close_count_);
+        delegate_->Close();
+    }
+
+ private:
+    bool closed_ = false;
+    std::unique_ptr<BatchReader> delegate_;
+    int32_t* close_count_;
+};
+
+}
+
 class MergedKeyValueRecordReaderTest : public testing::Test {
  public:
     void SetUp() override {
@@ -51,6 +110,14 @@ class MergedKeyValueRecordReaderTest : public testing::Test {
     std::shared_ptr<ReducerMergeFunctionWrapper> merge_function_wrapper_;
 };
 
+TEST_F(MergedKeyValueRecordReaderTest, TestRealtimeOffsetField) {
+    const DataField& field = RealtimeOffsetField();
+    ASSERT_EQ(std::numeric_limits<int32_t>::max() - 10002, field.Id());
+    ASSERT_EQ("_REALTIME_OFFSET", field.Name());
+    ASSERT_EQ(arrow::Type::INT64, field.Type()->id());
+    ASSERT_FALSE(field.Nullable());
+}
+
 TEST_F(MergedKeyValueRecordReaderTest, TestMergeAcrossUnderlyingBatches) {
     std::vector<DataField> fields = {DataField(0, arrow::field("k0", arrow::int32())),
                                      DataField(1, arrow::field("k1", arrow::int32())),
@@ -143,4 +210,335 @@ TEST_F(MergedKeyValueRecordReaderTest, TestSkipMergedNulloptResultInHasNext) {
     }
 }
 
+TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderOffsetFilter) {
+    std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32())),
+                                           DataField(1, arrow::field("v0", arrow::int32()))};
+    std::shared_ptr<arrow::Schema> value_schema =
+        DataField::ConvertDataFieldsToArrowSchema(value_fields);
+    std::shared_ptr<arrow::Schema> key_schema = arrow::schema({value_schema->field(0)});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema(value_schema->fields());
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
+        [0, 100, 0, 1, 10],
+        [0, 101, 1, 2, 20],
+        [0, 102, 4, 3, 30],
+        [0, 103, 2, 4, 40],
+        [0, 104, 5, 5, 50],
+        [0, 105, 3, 6, 60]
+    ])")
+            .ValueOrDie());
+
+    auto batch_reader =
+        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 2);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, OffsetRange(2, 4),
+                                 key_schema, value_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(
+        std::vector<KeyValue> results,
+        (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
+                                                    KeyValueRecordReader::Iterator>(reader.get())));
+
+    std::vector<RowKind*> row_kinds = {const_cast<RowKind*>(RowKind::Insert()),
+                                       const_cast<RowKind*>(RowKind::Insert())};
+    std::vector<int64_t> levels = {KeyValue::UNKNOWN_LEVEL, KeyValue::UNKNOWN_LEVEL};
+    std::vector<KeyValue> expected = KeyValueChecker::GenerateKeyValues(
+        row_kinds, {103, 105}, levels, {{4}, {6}}, {{4, 40}, {6, 60}}, pool_);
+    KeyValueChecker::CheckResult(expected, results, 1, 2);
+}
+
+TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderRawCountBeforeDedup) {
+    std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32())),
+                                           DataField(1, arrow::field("v0", arrow::int32()))};
+    std::shared_ptr<arrow::Schema> value_schema =
+        DataField::ConvertDataFieldsToArrowSchema(value_fields);
+    std::shared_ptr<arrow::Schema> key_schema = arrow::schema({value_schema->field(0)});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema(value_schema->fields());
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
+        [0, 10, 0, 1, 100],
+        [2, 11, 1, 1, 101],
+        [0, 12, 2, 2, 200]
+    ])")
+            .ValueOrDie());
+
+    int64_t raw_row_count = 0;
+    auto batch_reader =
+        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 2);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> raw_reader,
+        AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, std::nullopt, key_schema,
+                                 value_schema, pool_, &raw_row_count));
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<FieldsComparator> key_comparator,
+                         FieldsComparator::Create({value_fields[0]}, true));
+    auto merged_reader = std::make_unique<MergedKeyValueRecordReader>(
+        std::move(raw_reader), key_comparator, merge_function_wrapper_);
+    ASSERT_OK_AND_ASSIGN(
+        std::vector<KeyValue> results,
+        (ReadResultCollector::CollectKeyValueResult<
+            MergedKeyValueRecordReader, KeyValueRecordReader::Iterator>(merged_reader.get())));
+
+    ASSERT_EQ(raw_row_count, 3);
+    std::vector<RowKind*> row_kinds = {const_cast<RowKind*>(RowKind::UpdateAfter()),
+                                       const_cast<RowKind*>(RowKind::Insert())};
+    std::vector<int64_t> levels = {KeyValue::UNKNOWN_LEVEL, KeyValue::UNKNOWN_LEVEL};
+    std::vector<KeyValue> expected = KeyValueChecker::GenerateKeyValues(
+        row_kinds, {11, 12}, levels, {{1}, {2}}, {{1, 101}, {2, 200}}, pool_);
+    KeyValueChecker::CheckResult(expected, results, 1, 2);
+}
+
+TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderRawCountBeforeFilter) {
+    std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32()))};
+    std::shared_ptr<arrow::Schema> value_schema =
+        DataField::ConvertDataFieldsToArrowSchema(value_fields);
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema(value_schema->fields());
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
+        [0, 10, 0, 1],
+        [0, 11, 1, 2],
+        [0, 12, 2, 3],
+        [0, 13, 3, 4]
+    ])")
+            .ValueOrDie());
+
+    int64_t raw_row_count = 0;
+    auto batch_reader =
+        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 2);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, OffsetRange(2, 4),
+                                 value_schema, value_schema, pool_, &raw_row_count));
+    ASSERT_OK_AND_ASSIGN(
+        std::vector<KeyValue> results,
+        (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
+                                                    KeyValueRecordReader::Iterator>(reader.get())));
+
+    ASSERT_EQ(results.size(), 2);
+    ASSERT_EQ(raw_row_count, 4);
+}
+
+TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderCommitSchema) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Field> extra = MakeField("extra", arrow::int32(), 1);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key, extra});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1, 2]])")
+            .ValueOrDie());
+
+    auto query_batch_reader =
+        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> query_reader,
+        AdaptPreparedBatchReader(std::move(query_batch_reader), prepared_schema, OffsetRange(0, 1),
+                                 value_schema, value_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(
+        std::vector<KeyValue> query_results,
+        (ReadResultCollector::CollectKeyValueResult<
+            KeyValueRecordReader, KeyValueRecordReader::Iterator>(query_reader.get())));
+    ASSERT_EQ(query_results.size(), 1);
+    ASSERT_EQ(query_results[0].value->GetFieldCount(), 1);
+    ASSERT_EQ(query_results[0].value->GetInt(0), 1);
+
+    auto batch_reader =
+        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
+    ASSERT_NOK_WITH_MSG(AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema,
+                                                 std::nullopt, value_schema, value_schema, pool_),
+                        "exact");
+}
+
+TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderSafeDecode) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+
+    arrow::FieldVector invalid_fields = prepared_schema->fields();
+    invalid_fields[0] = invalid_fields[0]->WithName("wrong_value_kind");
+    invalid_fields[3] = MakeField("wrong_key", arrow::int32(), 99);
+    std::shared_ptr<arrow::DataType> invalid_type = arrow::struct_(invalid_fields);
+    auto invalid_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(invalid_type, R"([[0, 10, 0, 1]])").ValueOrDie());
+
+    auto batch_reader =
+        std::make_unique<MockFileBatchReader>(invalid_array, invalid_type, 1);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
+                                 value_schema, value_schema, pool_));
+    ASSERT_NOK_WITH_MSG(
+        (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
+                                                    KeyValueRecordReader::Iterator>(reader.get())),
+        "prepared batch field");
+}
+
+TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderNestedProjection) {
+    std::shared_ptr<arrow::Field> id = MakeField("id", arrow::int32(), 0);
+    std::shared_ptr<arrow::Field> item_a = MakeField("a", arrow::int32(), 10);
+    std::shared_ptr<arrow::Field> item_b = MakeField("b", arrow::int32(), 11);
+    std::shared_ptr<arrow::Field> items =
+        MakeField("items", arrow::list(arrow::field("item", arrow::struct_({item_a, item_b}))), 2);
+    std::shared_ptr<arrow::Field> attr_x = MakeField("x", arrow::int32(), 20);
+    std::shared_ptr<arrow::Field> attr_y = MakeField("y", arrow::int32(), 21);
+    std::shared_ptr<arrow::Field> attrs =
+        MakeField("attrs", arrow::map(arrow::utf8(), arrow::struct_({attr_x, attr_y})), 3);
+    std::shared_ptr<arrow::Field> key_left = MakeField("left", arrow::int32(), 30);
+    std::shared_ptr<arrow::Field> key_right = MakeField("right", arrow::int32(), 31);
+    std::shared_ptr<arrow::Field> keyed_values = MakeField(
+        "keyed_values", arrow::map(arrow::struct_({key_left, key_right}), arrow::int32()), 4);
+    std::shared_ptr<arrow::Schema> full_value_schema =
+        arrow::schema({id, items, attrs, keyed_values});
+    std::shared_ptr<arrow::Schema> key_schema = arrow::schema({id});
+    std::shared_ptr<arrow::Schema> prepared_schema =
+        MakePreparedSchema(full_value_schema->fields());
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
+        [0, 10, 0, 1, [[100, 200], [300, 400]], [["k1", [7, 8]], ["k2", [9, 10]]], [[[11, 12], 13], [[21, 22], 23]]]
+    ])")
+            .ValueOrDie());
+
+    std::shared_ptr<arrow::Field> query_item_b = MakeField("renamed_b", arrow::int32(), 11);
+    std::shared_ptr<arrow::Field> query_item_a = MakeField("renamed_a", arrow::int32(), 10);
+    std::shared_ptr<arrow::Field> query_items = MakeField(
+        "items_renamed",
+        arrow::list(arrow::field("element", arrow::struct_({query_item_b, query_item_a}))), 2);
+    std::shared_ptr<arrow::Field> query_attr_y = MakeField("renamed_y", arrow::int32(), 21);
+    std::shared_ptr<arrow::Field> query_attr_x = MakeField("renamed_x", arrow::int32(), 20);
+    std::shared_ptr<arrow::Field> query_attrs =
+        MakeField("attrs_renamed",
+                  arrow::map(arrow::utf8(), arrow::struct_({query_attr_y, query_attr_x})), 3);
+    std::shared_ptr<arrow::Field> query_key_right = MakeField("renamed_right", arrow::int32(), 31);
+    std::shared_ptr<arrow::Field> query_key_left = MakeField("renamed_left", arrow::int32(), 30);
+    std::shared_ptr<arrow::Field> query_keyed_values =
+        MakeField("keyed_values_renamed",
+                  arrow::map(arrow::struct_({query_key_right, query_key_left}), arrow::int32()), 4);
+    std::shared_ptr<arrow::Schema> query_value_schema =
+        arrow::schema({id, query_items, query_attrs, query_keyed_values});
+
+    auto batch_reader =
+        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
+                                 key_schema, query_value_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(
+        std::vector<KeyValue> results,
+        (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
+                                                    KeyValueRecordReader::Iterator>(reader.get())));
+
+    ASSERT_EQ(results.size(), 1);
+    ASSERT_EQ(results[0].key->GetInt(0), 1);
+    ASSERT_EQ(results[0].value->GetFieldCount(), 4);
+    ASSERT_EQ(results[0].value->GetInt(0), 1);
+
+    std::shared_ptr<InternalArray> item_array = results[0].value->GetArray(1);
+    ASSERT_EQ(item_array->Size(), 2);
+    std::shared_ptr<InternalRow> first_item = item_array->GetRow(0, 2);
+    ASSERT_EQ(first_item->GetInt(0), 200);
+    ASSERT_EQ(first_item->GetInt(1), 100);
+    std::shared_ptr<InternalRow> second_item = item_array->GetRow(1, 2);
+    ASSERT_EQ(second_item->GetInt(0), 400);
+    ASSERT_EQ(second_item->GetInt(1), 300);
+
+    std::shared_ptr<InternalMap> attr_map = results[0].value->GetMap(2);
+    ASSERT_EQ(attr_map->Size(), 2);
+    std::shared_ptr<InternalArray> key_array = attr_map->KeyArray();
+    ASSERT_EQ(std::string(key_array->GetStringView(0)), "k1");
+    ASSERT_EQ(std::string(key_array->GetStringView(1)), "k2");
+    std::shared_ptr<InternalArray> value_array = attr_map->ValueArray();
+    std::shared_ptr<InternalRow> first_attr = value_array->GetRow(0, 2);
+    ASSERT_EQ(first_attr->GetInt(0), 8);
+    ASSERT_EQ(first_attr->GetInt(1), 7);
+    std::shared_ptr<InternalRow> second_attr = value_array->GetRow(1, 2);
+    ASSERT_EQ(second_attr->GetInt(0), 10);
+    ASSERT_EQ(second_attr->GetInt(1), 9);
+
+    std::shared_ptr<InternalMap> keyed_value_map = results[0].value->GetMap(3);
+    ASSERT_EQ(keyed_value_map->Size(), 2);
+    std::shared_ptr<InternalArray> struct_keys = keyed_value_map->KeyArray();
+    std::shared_ptr<InternalRow> first_key = struct_keys->GetRow(0, 2);
+    ASSERT_EQ(first_key->GetInt(0), 12);
+    ASSERT_EQ(first_key->GetInt(1), 11);
+    std::shared_ptr<InternalRow> second_key = struct_keys->GetRow(1, 2);
+    ASSERT_EQ(second_key->GetInt(0), 22);
+    ASSERT_EQ(second_key->GetInt(1), 21);
+    ASSERT_EQ(keyed_value_map->ValueArray()->GetInt(0), 13);
+    ASSERT_EQ(keyed_value_map->ValueArray()->GetInt(1), 23);
+}
+
+TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderClose) {
+    std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32())),
+                                           DataField(1, arrow::field("v0", arrow::int32()))};
+    std::shared_ptr<arrow::Schema> value_schema =
+        DataField::ConvertDataFieldsToArrowSchema(value_fields);
+    std::shared_ptr<arrow::Schema> key_schema = arrow::schema({value_schema->field(0)});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema(value_schema->fields());
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
+        [0, 10, 0, 1, 100]
+    ])")
+            .ValueOrDie());
+
+    int32_t explicit_close_count = 0;
+    {
+        auto tracking_reader = std::make_unique<TrackingBatchReader>(
+            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1),
+            &explicit_close_count);
+        ASSERT_OK_AND_ASSIGN(
+            std::unique_ptr<KeyValueRecordReader> reader,
+            AdaptPreparedBatchReader(std::move(tracking_reader), prepared_schema, OffsetRange(0, 1),
+                                     key_schema, value_schema, pool_));
+        reader->Close();
+        reader->Close();
+    }
+    ASSERT_EQ(explicit_close_count, 1);
+
+    int32_t destructor_close_count = 0;
+    {
+        auto tracking_reader = std::make_unique<TrackingBatchReader>(
+            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1),
+            &destructor_close_count);
+        ASSERT_OK_AND_ASSIGN(
+            std::unique_ptr<KeyValueRecordReader> reader,
+            AdaptPreparedBatchReader(std::move(tracking_reader), prepared_schema, OffsetRange(0, 1),
+                                     key_schema, value_schema, pool_));
+    }
+    ASSERT_EQ(destructor_close_count, 1);
+
+    int32_t factory_failure_close_count = 0;
+    {
+        std::unique_ptr<BatchReader> tracking_reader = std::make_unique<TrackingBatchReader>(
+            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1),
+            &factory_failure_close_count);
+        std::shared_ptr<arrow::Schema> invalid_schema = arrow::schema(value_schema->fields());
+        ASSERT_NOK(AdaptPreparedBatchReader(std::move(tracking_reader), invalid_schema,
+                                            OffsetRange(0, 1), key_schema, value_schema, pool_));
+        ASSERT_EQ(nullptr, tracking_reader);
+    }
+    ASSERT_EQ(factory_failure_close_count, 1);
+
+    int32_t read_failure_close_count = 0;
+    {
+        auto failing_reader =
+            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
+        failing_reader->SetNextBatchStatus(Status::IOError("prepared reader failure"));
+        auto tracking_reader = std::make_unique<TrackingBatchReader>(std::move(failing_reader),
+                                                                     &read_failure_close_count);
+        ASSERT_OK_AND_ASSIGN(
+            std::unique_ptr<KeyValueRecordReader> reader,
+            AdaptPreparedBatchReader(std::move(tracking_reader), prepared_schema, OffsetRange(0, 1),
+                                     key_schema, value_schema, pool_));
+        ASSERT_NOK_WITH_MSG(reader->NextBatch(), "prepared reader failure");
+        ASSERT_EQ(read_failure_close_count, 1);
+        reader->Close();
+    }
+    ASSERT_EQ(read_failure_close_count, 1);
+}
+
 }  // namespace paimon::test
diff --git a/src/paimon/core/io/prepared_key_value_reader.cpp b/src/paimon/core/io/prepared_key_value_reader.cpp
new file mode 100644
index 000000000..0f4f22097
--- /dev/null
+++ b/src/paimon/core/io/prepared_key_value_reader.cpp
@@ -0,0 +1,565 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/io/prepared_key_value_reader.h"
+
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include "arrow/array/array_base.h"
+#include "arrow/array/array_nested.h"
+#include "arrow/array/array_primitive.h"
+#include "arrow/array/builder_primitive.h"
+#include "arrow/c/bridge.h"
+#include "arrow/compute/api.h"
+#include "arrow/type.h"
+#include "fmt/format.h"
+#include "paimon/common/data/columnar/columnar_batch_context.h"
+#include "paimon/common/data/columnar/columnar_row_ref.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/common/types/row_kind.h"
+#include "paimon/common/utils/arrow/arrow_utils.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/scope_guard.h"
+#include "paimon/core/realtime/realtime_fields.h"
+#include "paimon/core/utils/nested_projection_utils.h"
+#include "paimon/macros.h"
+#include "paimon/reader/batch_reader.h"
+#include "paimon/status.h"
+
+namespace paimon {
+
+namespace {
+
+constexpr int32_t kValueKindIndex = 0;
+constexpr int32_t kSequenceNumberIndex = 1;
+constexpr int32_t kRealtimeOffsetIndex = 2;
+constexpr int32_t kPreparedValueStartIndex = 3;
+
+Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
+    const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& read_type);
+
+Status CheckPreparedField(const std::shared_ptr<arrow::Schema>& schema, int32_t field_idx,
+                          const DataField& expected_field) {
+    if (schema->num_fields() <= field_idx) {
+        return Status::Invalid(fmt::format("prepared schema missing transport field {} at index {}",
+                                           expected_field.Name(), field_idx));
+    }
+    const std::shared_ptr<arrow::Field>& field = schema->field(field_idx);
+    PAIMON_ASSIGN_OR_RAISE(int32_t field_id, NestedProjectionUtils::GetPaimonFieldId(field));
+    if (field->name() != expected_field.Name() || !field->type()->Equals(*expected_field.Type()) ||
+        field->nullable() || field_id != expected_field.Id()) {
+        return Status::Invalid(fmt::format(
+            "prepared schema field {} must be non-null {}:{} with field id {}, got {}:{} "
+            "nullable={} field id {}",
+            field_idx, expected_field.Name(), expected_field.Type()->ToString(),
+            expected_field.Id(), field->name(), field->type()->ToString(), field->nullable(),
+            field_id));
+    }
+    return Status::OK();
+}
+
+Result<int32_t> FindFieldIndexByPaimonId(const arrow::FieldVector& fields, int32_t field_id) {
+    std::optional<int32_t> matching_index;
+    for (int32_t i = 0; i < static_cast<int32_t>(fields.size()); ++i) {
+        PAIMON_ASSIGN_OR_RAISE(int32_t candidate_id,
+                               NestedProjectionUtils::GetPaimonFieldId(fields[i]));
+        if (candidate_id == field_id) {
+            if (matching_index.has_value()) {
+                return Status::Invalid(
+                    fmt::format("duplicate field id {} in prepared schema", field_id));
+            }
+            matching_index = i;
+        }
+    }
+    if (matching_index.has_value()) {
+        return matching_index.value();
+    }
+    return Status::Invalid(fmt::format("cannot find field id {} in prepared schema", field_id));
+}
+
+Status ValidateProjectionType(const std::shared_ptr<arrow::DataType>& prepared_type,
+                              const std::shared_ptr<arrow::DataType>& query_type) {
+    if (prepared_type->id() != query_type->id()) {
+        return Status::Invalid(fmt::format("prepared value type {} does not match query type {}",
+                                           prepared_type->ToString(), query_type->ToString()));
+    }
+    switch (query_type->id()) {
+        case arrow::Type::STRUCT: {
+            const arrow::FieldVector& prepared_fields = prepared_type->fields();
+            for (const std::shared_ptr<arrow::Field>& query_field : query_type->fields()) {
+                PAIMON_ASSIGN_OR_RAISE(int32_t query_id,
+                                       NestedProjectionUtils::GetPaimonFieldId(query_field));
+                PAIMON_ASSIGN_OR_RAISE(int32_t prepared_idx,
+                                       FindFieldIndexByPaimonId(prepared_fields, query_id));
+                PAIMON_RETURN_NOT_OK(ValidateProjectionType(prepared_fields[prepared_idx]->type(),
+                                                            query_field->type()));
+            }
+            return Status::OK();
+        }
+        case arrow::Type::LIST:
+            return ValidateProjectionType(prepared_type->field(0)->type(),
+                                          query_type->field(0)->type());
+        case arrow::Type::MAP: {
+            const std::shared_ptr<arrow::MapType> prepared_map =
+                checked_pointer_cast<arrow::MapType>(prepared_type);
+            const std::shared_ptr<arrow::MapType> query_map =
+                checked_pointer_cast<arrow::MapType>(query_type);
+            PAIMON_RETURN_NOT_OK(
+                ValidateProjectionType(prepared_map->key_type(), query_map->key_type()));
+            return ValidateProjectionType(prepared_map->item_type(), query_map->item_type());
+        }
+        default:
+            if (!prepared_type->Equals(*query_type)) {
+                return Status::Invalid(
+                    fmt::format("prepared leaf type {} does not match query type {}",
+                                prepared_type->ToString(), query_type->ToString()));
+            }
+            return Status::OK();
+    }
+}
+
+Status ValidateProjectionSchema(const std::shared_ptr<arrow::Schema>& prepared_schema,
+                                const std::shared_ptr<arrow::Schema>& query_schema) {
+    arrow::FieldVector prepared_value_fields(
+        prepared_schema->fields().begin() + kPreparedValueStartIndex,
+        prepared_schema->fields().end());
+    for (const std::shared_ptr<arrow::Field>& query_field : query_schema->fields()) {
+        PAIMON_ASSIGN_OR_RAISE(int32_t query_id,
+                               NestedProjectionUtils::GetPaimonFieldId(query_field));
+        PAIMON_ASSIGN_OR_RAISE(int32_t prepared_idx,
+                               FindFieldIndexByPaimonId(prepared_value_fields, query_id));
+        PAIMON_RETURN_NOT_OK(ValidateProjectionType(prepared_value_fields[prepared_idx]->type(),
+                                                    query_field->type()));
+    }
+    return Status::OK();
+}
+
+Status ValidateExactCommitSchema(const std::shared_ptr<arrow::Schema>& prepared_schema,
+                                 const std::shared_ptr<arrow::Schema>& value_schema) {
+    if (prepared_schema->num_fields() != value_schema->num_fields() + kPreparedValueStartIndex) {
+        return Status::Invalid("commit requires the exact prepared writer schema");
+    }
+    for (int32_t i = 0; i < value_schema->num_fields(); ++i) {
+        if (!prepared_schema->field(i + kPreparedValueStartIndex)
+                 ->Equals(value_schema->field(i), true)) {
+            return Status::Invalid("commit requires the exact prepared writer schema");
+        }
+    }
+    return Status::OK();
+}
+
+Status ValidatePreparedSchema(const std::shared_ptr<arrow::Schema>& prepared_schema) {
+    if (!prepared_schema || prepared_schema->num_fields() < kPreparedValueStartIndex) {
+        return Status::Invalid("prepared schema must contain realtime transport fields");
+    }
+    PAIMON_RETURN_NOT_OK(
+        CheckPreparedField(prepared_schema, kValueKindIndex, SpecialFields::ValueKind()));
+    PAIMON_RETURN_NOT_OK(
+        CheckPreparedField(prepared_schema, kSequenceNumberIndex, SpecialFields::SequenceNumber()));
+    PAIMON_RETURN_NOT_OK(
+        CheckPreparedField(prepared_schema, kRealtimeOffsetIndex, RealtimeOffsetField()));
+    return Status::OK();
+}
+
+Result<std::shared_ptr<arrow::Array>> AlignStructArrayByPaimonIds(
+    const std::shared_ptr<arrow::StructArray>& array,
+    const std::shared_ptr<arrow::StructType>& read_type) {
+    const std::shared_ptr<arrow::StructType> data_type =
+        checked_pointer_cast<arrow::StructType>(array->type());
+    std::unordered_map<int32_t, int32_t> data_field_id_to_idx;
+    data_field_id_to_idx.reserve(data_type->num_fields());
+    for (int32_t i = 0; i < data_type->num_fields(); ++i) {
+        PAIMON_ASSIGN_OR_RAISE(int32_t field_id,
+                               NestedProjectionUtils::GetPaimonFieldId(data_type->field(i)));
+        if (!data_field_id_to_idx.emplace(field_id, i).second) {
+            return Status::Invalid(
+                fmt::format("duplicate field id {} in prepared value struct", field_id));
+        }
+    }
+
+    arrow::ArrayVector aligned_arrays;
+    aligned_arrays.reserve(read_type->num_fields());
+    for (const std::shared_ptr<arrow::Field>& read_field : read_type->fields()) {
+        PAIMON_ASSIGN_OR_RAISE(int32_t read_field_id,
+                               NestedProjectionUtils::GetPaimonFieldId(read_field));
+        auto data_iter = data_field_id_to_idx.find(read_field_id);
+        if (data_iter == data_field_id_to_idx.end()) {
+            return Status::Invalid(
+                fmt::format("cannot find field id {} in prepared value struct", read_field_id));
+        }
+        std::shared_ptr<arrow::Array> child = array->field(data_iter->second);
+        PAIMON_ASSIGN_OR_RAISE(child, AlignArrayByPaimonIds(child, read_field->type()));
+        aligned_arrays.push_back(std::move(child));
+    }
+
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+        std::shared_ptr<arrow::StructArray> aligned,
+        arrow::StructArray::Make(aligned_arrays, read_type->fields(), array->null_bitmap(),
+                                 array->null_count(), array->offset()));
+    return aligned;
+}
+
+Result<std::shared_ptr<arrow::Array>> AlignListArrayByPaimonIds(
+    const std::shared_ptr<arrow::ListArray>& array,
+    const std::shared_ptr<arrow::ListType>& read_type) {
+    std::shared_ptr<arrow::Array> values = array->values();
+    PAIMON_ASSIGN_OR_RAISE(values, AlignArrayByPaimonIds(values, read_type->value_type()));
+    std::shared_ptr<arrow::ArrayData> new_data = array->data()->Copy();
+    new_data->type = read_type;
+    new_data->child_data = {values->data()};
+    return arrow::MakeArray(new_data);
+}
+
+Result<std::shared_ptr<arrow::Array>> AlignMapArrayByPaimonIds(
+    const std::shared_ptr<arrow::MapArray>& array,
+    const std::shared_ptr<arrow::MapType>& read_type) {
+    std::shared_ptr<arrow::Array> keys = array->keys();
+    PAIMON_ASSIGN_OR_RAISE(keys, AlignArrayByPaimonIds(keys, read_type->key_type()));
+    std::shared_ptr<arrow::Array> items = array->items();
+    PAIMON_ASSIGN_OR_RAISE(items, AlignArrayByPaimonIds(items, read_type->item_type()));
+
+    const std::shared_ptr<arrow::ArrayData>& entries_data = array->data()->child_data[0];
+    std::shared_ptr<arrow::ArrayData> new_entries = entries_data->Copy();
+    new_entries->type = arrow::struct_({read_type->key_field(), read_type->item_field()});
+    new_entries->child_data = {keys->data(), items->data()};
+
+    std::shared_ptr<arrow::ArrayData> new_data = array->data()->Copy();
+    new_data->type = read_type;
+    new_data->child_data = {std::move(new_entries)};
+    return arrow::MakeArray(new_data);
+}
+
+Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
+    const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& read_type) {
+    if (array->type()->id() != read_type->id()) {
+        return Status::Invalid(fmt::format("prepared value type {} does not match query type {}",
+                                           array->type()->ToString(), read_type->ToString()));
+    }
+    switch (read_type->id()) {
+        case arrow::Type::STRUCT:
+            return AlignStructArrayByPaimonIds(checked_pointer_cast<arrow::StructArray>(array),
+                                               checked_pointer_cast<arrow::StructType>(read_type));
+        case arrow::Type::LIST:
+            return AlignListArrayByPaimonIds(checked_pointer_cast<arrow::ListArray>(array),
+                                             checked_pointer_cast<arrow::ListType>(read_type));
+        case arrow::Type::MAP:
+            return AlignMapArrayByPaimonIds(checked_pointer_cast<arrow::MapArray>(array),
+                                            checked_pointer_cast<arrow::MapType>(read_type));
+        default:
+            if (!array->type()->Equals(*read_type)) {
+                return Status::Invalid(
+                    fmt::format("prepared leaf type {} does not match query type {}",
+                                array->type()->ToString(), read_type->ToString()));
+            }
+            return array;
+    }
+}
+
+Result<arrow::ArrayVector> ProjectFieldsByPaimonIds(
+    const std::shared_ptr<arrow::StructArray>& data_batch,
+    const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::shared_ptr<arrow::Schema>& query_schema) {
+    std::unordered_map<int32_t, int32_t> prepared_field_id_to_idx;
+    prepared_field_id_to_idx.reserve(prepared_schema->num_fields());
+    for (int32_t i = kPreparedValueStartIndex; i < prepared_schema->num_fields(); ++i) {
+        PAIMON_ASSIGN_OR_RAISE(int32_t field_id,
+                               NestedProjectionUtils::GetPaimonFieldId(prepared_schema->field(i)));
+        if (!prepared_field_id_to_idx.emplace(field_id, i).second) {
+            return Status::Invalid(
+                fmt::format("duplicate field id {} in prepared schema", field_id));
+        }
+    }
+
+    arrow::ArrayVector result;
+    result.reserve(query_schema->num_fields());
+    for (const std::shared_ptr<arrow::Field>& query_field : query_schema->fields()) {
+        PAIMON_ASSIGN_OR_RAISE(int32_t query_field_id,
+                               NestedProjectionUtils::GetPaimonFieldId(query_field));
+        auto prepared_iter = prepared_field_id_to_idx.find(query_field_id);
+        if (prepared_iter == prepared_field_id_to_idx.end()) {
+            return Status::Invalid(
+                fmt::format("cannot find field id {} in prepared schema", query_field_id));
+        }
+        std::shared_ptr<arrow::Array> field_array = data_batch->field(prepared_iter->second);
+        PAIMON_ASSIGN_OR_RAISE(field_array,
+                               AlignArrayByPaimonIds(field_array, query_field->type()));
+        result.push_back(std::move(field_array));
+    }
+    return result;
+}
+
+Result<std::shared_ptr<arrow::StructArray>> ApplyOffsetFilter(
+    const std::shared_ptr<arrow::StructArray>& data_batch,
+    const std::shared_ptr<arrow::NumericArray<arrow::Int64Type>>& offset_array,
+    const std::optional<OffsetRange>& visible_offsets, arrow::MemoryPool* arrow_pool) {
+    if (!visible_offsets.has_value()) {
+        return data_batch;
+    }
+
+    arrow::BooleanBuilder filter_builder(arrow_pool);
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(filter_builder.Reserve(offset_array->length()));
+    int64_t visible_row_count = 0;
+    for (int64_t i = 0; i < offset_array->length(); ++i) {
+        int64_t offset = offset_array->Value(i);
+        bool visible = offset >= visible_offsets->begin && offset < visible_offsets->end;
+        filter_builder.UnsafeAppend(visible);
+        visible_row_count += visible;
+    }
+    if (visible_row_count == 0) {
+        return std::shared_ptr<arrow::StructArray>();
+    }
+    if (visible_row_count == data_batch->length()) {
+        return data_batch;
+    }
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> filter,
+                                      filter_builder.Finish());
+    arrow::compute::ExecContext exec_context(arrow_pool);
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+        arrow::Datum filtered,
+        arrow::compute::Filter(data_batch, filter, arrow::compute::FilterOptions::Defaults(),
+                               &exec_context));
+    return checked_pointer_cast<arrow::StructArray>(filtered.make_array());
+}
+
+class PreparedKeyValueReader final : public KeyValueRecordReader {
+ public:
+    PreparedKeyValueReader(std::unique_ptr<BatchReader>&& reader,
+                           const std::shared_ptr<arrow::Schema>& prepared_schema,
+                           const std::optional<OffsetRange>& visible_offsets,
+                           const std::shared_ptr<arrow::Schema>& key_schema,
+                           const std::shared_ptr<arrow::Schema>& value_schema,
+                           const std::shared_ptr<MemoryPool>& pool, int64_t* raw_row_count)
+        : reader_(std::move(reader)),
+          prepared_schema_(prepared_schema),
+          visible_offsets_(visible_offsets),
+          key_schema_(key_schema),
+          value_schema_(value_schema),
+          pool_(pool),
+          arrow_pool_(GetArrowPool(pool)),
+          raw_row_count_(raw_row_count) {}
+
+    ~PreparedKeyValueReader() override {
+        Close();
+    }
+
+    class Iterator final : public KeyValueRecordReader::Iterator {
+     public:
+        explicit Iterator(PreparedKeyValueReader* reader) : reader_(reader) {}
+
+        Result<bool> HasNext() const override {
+            return cursor_ < reader_->row_kind_array_->length();
+        }
+
+        Result<KeyValue> Next() override {
+            if (cursor_ >= reader_->row_kind_array_->length()) {
+                return Status::Invalid("No more prepared key values in current iterator");
+            }
+            std::shared_ptr<InternalRow> key =
+                std::make_shared<ColumnarRowRef>(reader_->key_ctx_, cursor_);
+            auto value = std::make_unique<ColumnarRowRef>(reader_->value_ctx_, cursor_);
+            PAIMON_ASSIGN_OR_RAISE(
+                const RowKind* row_kind,
+                RowKind::FromByteValue(reader_->row_kind_array_->Value(cursor_)));
+            int64_t sequence_number = reader_->sequence_number_array_->Value(cursor_);
+            ++cursor_;
+            return KeyValue(row_kind, sequence_number, KeyValue::UNKNOWN_LEVEL, std::move(key),
+                            std::move(value));
+        }
+
+     private:
+        PreparedKeyValueReader* reader_;
+        int64_t cursor_ = 0;
+    };
+
+    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatch() override {
+        Result<std::unique_ptr<KeyValueRecordReader::Iterator>> result = NextBatchImpl();
+        if (!result.ok()) {
+            Close();
+        }
+        return result;
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return reader_->GetReaderMetrics();
+    }
+
+    void Close() override {
+        if (closed_) {
+            return;
+        }
+        closed_ = true;
+        ResetBatchState();
+        reader_->Close();
+    }
+
+ private:
+    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatchImpl() {
+        if (closed_) {
+            return std::unique_ptr<KeyValueRecordReader::Iterator>();
+        }
+
+        while (true) {
+            ResetBatchState();
+            PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader_->NextBatch());
+            if (BatchReader::IsEofBatch(batch)) {
+                return std::unique_ptr<KeyValueRecordReader::Iterator>();
+            }
+            auto& [c_array, c_schema] = batch;
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> arrow_array,
+                                              arrow::ImportArray(c_array.get(), c_schema.get()));
+            if (!arrow_array || arrow_array->type_id() != arrow::Type::STRUCT) {
+                return Status::Invalid("cannot cast prepared batch to StructArray");
+            }
+            std::shared_ptr<arrow::StructArray> data_batch =
+                checked_pointer_cast<arrow::StructArray>(arrow_array);
+            PAIMON_RETURN_NOT_OK(ValidatePreparedBatch(data_batch));
+            if (raw_row_count_ != nullptr) {
+                int64_t updated_count = 0;
+                if (__builtin_add_overflow(*raw_row_count_, data_batch->length(), &updated_count)) {
+                    return Status::Invalid("prepared raw row count overflow");
+                }
+                *raw_row_count_ = updated_count;
+            }
+
+            std::shared_ptr<arrow::NumericArray<arrow::Int64Type>> offset_array =
+                checked_pointer_cast<arrow::NumericArray<arrow::Int64Type>>(
+                    data_batch->field(kRealtimeOffsetIndex));
+            PAIMON_ASSIGN_OR_RAISE(
+                data_batch,
+                ApplyOffsetFilter(data_batch, offset_array, visible_offsets_, arrow_pool_.get()));
+            if (!data_batch) {
+                continue;
+            }
+
+            row_kind_array_ = checked_pointer_cast<arrow::NumericArray<arrow::Int8Type>>(
+                data_batch->field(kValueKindIndex));
+            sequence_number_array_ = checked_pointer_cast<arrow::NumericArray<arrow::Int64Type>>(
+                data_batch->field(kSequenceNumberIndex));
+            PAIMON_ASSIGN_OR_RAISE(
+                arrow::ArrayVector key_fields,
+                ProjectFieldsByPaimonIds(data_batch, prepared_schema_, key_schema_));
+            PAIMON_ASSIGN_OR_RAISE(
+                arrow::ArrayVector value_fields,
+                ProjectFieldsByPaimonIds(data_batch, prepared_schema_, value_schema_));
+            key_ctx_ = std::make_shared<ColumnarBatchContext>(key_fields, pool_);
+            value_ctx_ = std::make_shared<ColumnarBatchContext>(value_fields, pool_);
+            ArrowUtils::TraverseArray(data_batch);
+            return std::make_unique<Iterator>(this);
+        }
+    }
+
+    Status ValidatePreparedBatch(const std::shared_ptr<arrow::StructArray>& data_batch) const {
+        if (data_batch->num_fields() != prepared_schema_->num_fields()) {
+            return Status::Invalid(fmt::format(
+                "prepared batch field count {} does not match prepared schema field count {}",
+                data_batch->num_fields(), prepared_schema_->num_fields()));
+        }
+        const arrow::FieldVector& batch_fields = data_batch->type()->fields();
+        for (int32_t i = 0; i < data_batch->num_fields(); ++i) {
+            if (!batch_fields[i]->Equals(prepared_schema_->field(i), true)) {
+                return Status::Invalid(fmt::format(
+                    "prepared batch field {} does not match declared prepared schema", i));
+            }
+        }
+        if (!data_batch->field(kValueKindIndex) ||
+            data_batch->field(kValueKindIndex)->type_id() != arrow::Type::INT8) {
+            return Status::Invalid("cannot cast VALUE_KIND column to int8 arrow array");
+        }
+        if (!data_batch->field(kSequenceNumberIndex) ||
+            data_batch->field(kSequenceNumberIndex)->type_id() != arrow::Type::INT64) {
+            return Status::Invalid("cannot cast SEQUENCE_NUMBER column to int64 arrow array");
+        }
+        if (!data_batch->field(kRealtimeOffsetIndex) ||
+            data_batch->field(kRealtimeOffsetIndex)->type_id() != arrow::Type::INT64) {
+            return Status::Invalid("cannot cast REALTIME_OFFSET column to int64 arrow array");
+        }
+        if (data_batch->field(kValueKindIndex)->null_count() != 0 ||
+            data_batch->field(kSequenceNumberIndex)->null_count() != 0 ||
+            data_batch->field(kRealtimeOffsetIndex)->null_count() != 0) {
+            return Status::Invalid("prepared transport columns must not contain nulls");
+        }
+        return Status::OK();
+    }
+
+    void ResetBatchState() {
+        key_ctx_.reset();
+        value_ctx_.reset();
+        row_kind_array_.reset();
+        sequence_number_array_.reset();
+    }
+
+ private:
+    bool closed_ = false;
+    std::unique_ptr<BatchReader> reader_;
+    std::shared_ptr<arrow::Schema> prepared_schema_;
+    std::optional<OffsetRange> visible_offsets_;
+    std::shared_ptr<arrow::Schema> key_schema_;
+    std::shared_ptr<arrow::Schema> value_schema_;
+    std::shared_ptr<MemoryPool> pool_;
+    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
+    int64_t* raw_row_count_;
+    std::shared_ptr<ColumnarBatchContext> key_ctx_;
+    std::shared_ptr<ColumnarBatchContext> value_ctx_;
+    std::shared_ptr<arrow::NumericArray<arrow::Int8Type>> row_kind_array_;
+    std::shared_ptr<arrow::NumericArray<arrow::Int64Type>> sequence_number_array_;
+};
+
+}
+
+Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
+    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::optional<OffsetRange>& visible_offsets,
+    const std::shared_ptr<arrow::Schema>& key_schema,
+    const std::shared_ptr<arrow::Schema>& value_schema,
+    const std::shared_ptr<MemoryPool>& memory_pool, int64_t* raw_row_count) {
+    std::unique_ptr<BatchReader> owned_reader = std::move(reader);
+    if (!owned_reader) {
+        return Status::Invalid("prepared batch reader cannot be null");
+    }
+    ScopeGuard close_guard([&owned_reader]() -> void { owned_reader->Close(); });
+    PAIMON_RETURN_NOT_OK(ValidatePreparedSchema(prepared_schema));
+    if (!key_schema) {
+        return Status::Invalid("prepared key schema cannot be null");
+    }
+    if (!value_schema) {
+        return Status::Invalid("prepared value schema cannot be null");
+    }
+    if (!memory_pool) {
+        return Status::Invalid("prepared reader memory pool cannot be null");
+    }
+    PAIMON_RETURN_NOT_OK(ValidateProjectionSchema(prepared_schema, key_schema));
+    PAIMON_RETURN_NOT_OK(ValidateProjectionSchema(prepared_schema, value_schema));
+    if (!visible_offsets.has_value()) {
+        PAIMON_RETURN_NOT_OK(ValidateExactCommitSchema(prepared_schema, value_schema));
+    }
+    std::unique_ptr<KeyValueRecordReader> result(
+        new PreparedKeyValueReader(std::move(owned_reader), prepared_schema, visible_offsets,
+                                   key_schema, value_schema, memory_pool, raw_row_count));
+    close_guard.Release();
+    return result;
+}
+
+}
diff --git a/src/paimon/core/io/prepared_key_value_reader.h b/src/paimon/core/io/prepared_key_value_reader.h
new file mode 100644
index 000000000..e7a6f9651
--- /dev/null
+++ b/src/paimon/core/io/prepared_key_value_reader.h
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <optional>
+
+#include "arrow/type_fwd.h"
+#include "paimon/core/io/key_value_record_reader.h"
+#include "paimon/realtime/offset_range.h"
+#include "paimon/result.h"
+
+namespace paimon {
+class BatchReader;
+class MemoryPool;
+
+Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
+    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::optional<OffsetRange>& visible_offsets,
+    const std::shared_ptr<arrow::Schema>& key_schema,
+    const std::shared_ptr<arrow::Schema>& value_schema,
+    const std::shared_ptr<MemoryPool>& memory_pool, int64_t* raw_row_count = nullptr);
+
+}
diff --git a/src/paimon/core/realtime/realtime_fields.h b/src/paimon/core/realtime/realtime_fields.h
new file mode 100644
index 000000000..6ed04b38a
--- /dev/null
+++ b/src/paimon/core/realtime/realtime_fields.h
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <limits>
+
+#include "arrow/type.h"
+#include "paimon/common/types/data_field.h"
+
+namespace paimon {
+
+inline const DataField& RealtimeOffsetField() {
+    static const DataField data_field =
+        DataField(std::numeric_limits<int32_t>::max() - 10002,
+                  arrow::field("_REALTIME_OFFSET", arrow::int64(), /*nullable=*/false));
+    return data_field;
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp
index 47603497b..050f09701 100644
--- a/src/paimon/core/schema/schema_validation_test.cpp
+++ b/src/paimon/core/schema/schema_validation_test.cpp
@@ -46,6 +46,13 @@ TEST(SchemaValidationTest, TestSimple) {
     ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema));
 }
 
+TEST(SchemaValidationTest, TestRealtimeOffsetIsNotGloballyReserved) {
+    auto schema = arrow::schema({arrow::field("_REALTIME_OFFSET", arrow::int64())});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<TableSchema> table_schema,
+                         TableSchema::Create(0, schema, {}, {}, {}));
+    ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema));
+}
+
 TEST(SchemaValidationTest, TestVectorType) {
     auto vector_field = arrow::field("embedding", arrow::fixed_size_list(arrow::float32(), 3));
     auto schema = arrow::schema({arrow::field("id", arrow::int64()), vector_field});

From 232d587093d89c6320d4e107a0b9772040cd3d72 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Mon, 24 Aug 2026 11:26:17 +0800
Subject: [PATCH 15/62] refactor(realtime): prepare primary-key batches in
 framework

---
 include/paimon/realtime/realtime_context.h    |   4 +
 include/paimon/realtime/realtime_store.h      |  52 +-
 .../core/mergetree/merge_tree_writer_test.cpp |  15 +
 src/paimon/core/mergetree/write_buffer.cpp    |   4 +
 .../operation/key_value_file_store_write.cpp  |  40 +-
 .../key_value_file_store_write_test.cpp       | 374 +++++++++++-
 .../core/operation/merge_file_split_read.cpp  |  10 +-
 .../core/operation/merge_file_split_read.h    |   1 -
 .../realtime/arrow_realtime_store_factory.cpp |  32 +-
 .../realtime/primary_key_realtime_store.cpp   | 548 ++++--------------
 .../realtime/primary_key_realtime_store.h     |  27 +-
 .../primary_key_realtime_store_test.cpp       | 502 +++-------------
 .../core/realtime/realtime_context_impl.cpp   |  41 +-
 .../core/realtime/realtime_context_impl.h     |   5 -
 .../core/realtime/realtime_context_test.cpp   | 133 ++---
 .../realtime/realtime_primary_key_writer.cpp  | 332 +++++++----
 .../realtime/realtime_primary_key_writer.h    |  35 +-
 .../table/source/key_value_table_read.cpp     | 193 ++----
 test/inte/realtime_write_inte_test.cpp        | 239 ++++----
 19 files changed, 1106 insertions(+), 1481 deletions(-)

diff --git a/include/paimon/realtime/realtime_context.h b/include/paimon/realtime/realtime_context.h
index 200e4ba4c..8f2967b32 100644
--- a/include/paimon/realtime/realtime_context.h
+++ b/include/paimon/realtime/realtime_context.h
@@ -78,6 +78,10 @@ using RealtimeOffsetMap = std::map<RealtimePartitionBucket, int64_t>;
 /// partition drop, and rollback operations do not automatically clear process-local real-time
 /// state. Applications must coordinate these operations with active real-time writers and recreate
 /// the `RealtimeContext` and writers before continuing.
+///
+/// A primary-key writer and its context form one lifecycle. After a primary-key write or prepare
+/// returns an error, discard both, create fresh instances from the latest committed snapshot, and
+/// replay the caller-owned WAL. Reusing the failed context or retrying prepare is unsupported.
 class PAIMON_EXPORT RealtimeContext {
  public:
     /// Creates a context backed by Paimon's default in-memory Arrow `RealtimeStore`.
diff --git a/include/paimon/realtime/realtime_store.h b/include/paimon/realtime/realtime_store.h
index 1e53c173e..dc5d543ac 100644
--- a/include/paimon/realtime/realtime_store.h
+++ b/include/paimon/realtime/realtime_store.h
@@ -47,19 +47,15 @@ struct PAIMON_EXPORT AppendRealtimeStoreCreateConfig {
     StatisticsMode statistics_mode;
 };
 
-struct PAIMON_EXPORT PrimaryKeyRealtimeStoreCreateConfig {
-    std::vector<std::string> primary_keys;
-    /// Largest sequence restored from the committed snapshot. A PK store assigns one contiguous
-    /// sequence to every mutation in `Write` order, starting at the next value, and rejects
-    /// `Write` before the assigned sequence would exceed `INT64_MAX - 1`.
-    int64_t restore_max_sequence_number;
-};
+struct PAIMON_EXPORT PrimaryKeyRealtimeStoreCreateConfig {};
 
 using RealtimeStoreCreateConfig =
     std::variant<AppendRealtimeStoreCreateConfig, PrimaryKeyRealtimeStoreCreateConfig>;
 
 struct PAIMON_EXPORT RealtimeStoreCreateRequest {
-    /// Complete table write schema whose ownership is transferred to the factory.
+    /// Schema whose ownership is transferred to the factory. Append mode receives the complete
+    /// table write schema. Primary-key mode receives the prepared transport schema:
+    /// [_VALUE_KIND, _SEQUENCE_NUMBER, _REALTIME_OFFSET, table write fields].
     std::unique_ptr<::ArrowSchema> write_schema;
     std::map<std::string, std::string> options;
     std::shared_ptr<MemoryPool> memory_pool;
@@ -68,10 +64,12 @@ struct PAIMON_EXPORT RealtimeStoreCreateRequest {
     RealtimeStoreCreateConfig mode_config;
 };
 
-/// A table record batch and its framework-assigned contiguous offset range.
+/// A record batch and its framework-assigned contiguous offset range.
 ///
-/// The batch contains only table write fields. Row `i` is associated with
-/// `offset_range.begin + i`; the offset is progress metadata and is not a table field.
+/// Append-mode batches contain table write fields, and row `i` is associated with
+/// `offset_range.begin + i`. Primary-key batches contain the prepared transport schema supplied
+/// to the factory and are physically sorted by full primary key then sequence number; their
+/// per-row `_REALTIME_OFFSET` field preserves the original write-order offset after sorting.
 struct PAIMON_EXPORT RealtimeWriteBatch {
     /// Input batch whose ownership is transferred to `RealtimeStore::Write`.
     std::unique_ptr<RecordBatch> batch;
@@ -106,7 +104,8 @@ class PAIMON_EXPORT RealtimeReadView {
 
 /// Parameters used by a `RealtimeStore` to create readers for a query.
 struct PAIMON_EXPORT RealtimeQueryContext {
-    /// Requested output fields before the mandatory leading `_VALUE_KIND` field is added.
+    /// Append mode receives the requested output fields before the mandatory leading
+    /// `_VALUE_KIND` field is added. Primary-key mode receives the complete prepared schema.
     ::ArrowSchema* read_schema;
     /// Predicate using field indexes from `read_schema`.
     std::shared_ptr<Predicate> predicate;
@@ -143,9 +142,13 @@ class PAIMON_EXPORT RealtimeStore {
 
     /// Creates readers that expose all rows in a sealed segment for Paimon file writing.
     ///
-    /// Concatenating the returned readers must produce every sealed row exactly once and in write
-    /// order. Each output batch contains `_VALUE_KIND` followed by all fields from the factory's
-    /// `write_schema`.
+    /// Concatenating the returned readers must produce every sealed row exactly once. Append-mode
+    /// readers preserve write order and contain `_VALUE_KIND` followed by table write fields.
+    /// Primary-key readers expose raw prepared rows. Each returned reader's complete stream,
+    /// including across `NextBatch` boundaries, is sorted by full primary key then sequence
+    /// number; all readers collectively cover sealed mutations exactly once. Reader cardinality is
+    /// independent of the number of writes. Paimon adapts and merges those rows before writing
+    /// files.
     virtual Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
         const std::shared_ptr<RealtimeSegmentHandle>& segment) = 0;
 
@@ -155,16 +158,17 @@ class PAIMON_EXPORT RealtimeStore {
     /// also provide a consistent snapshot when a write or seal is in progress.
     virtual Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() = 0;
 
-    /// Creates readers over rows in `view` whose offsets are greater than or equal to
-    /// `offset_begin`.
+    /// Creates readers over rows in `view`. Append mode returns rows whose offsets are greater
+    /// than or equal to `offset_begin`. Primary-key mode ignores `offset_begin` and returns raw
+    /// prepared rows; Paimon applies offset filtering, projection, and merge-on-read adaptation.
     ///
-    /// Each output batch contains `_VALUE_KIND` first, followed by the fields requested by
-    /// `context.read_schema` except a duplicate `_VALUE_KIND`. Concatenating all returned readers
-    /// must produce every matching row once. Primary-key readers additionally provide a non-null
-    /// `_SEQUENCE_NUMBER` when requested, are individually sorted by primary key, and contain at
-    /// most one mutation per key. Assigned sequences remain stable across views and queries;
-    /// readers need not be globally sorted with one another. Paimon retains `view` for the lifetime
-    /// of the resulting framework reader.
+    /// Append-mode output batches contain `_VALUE_KIND` first, followed by requested fields except
+    /// a duplicate `_VALUE_KIND`; all returned append readers collectively cover every matching
+    /// row exactly once. Primary-key output batches use the prepared transport schema and may
+    /// contain multiple mutations per key. Each returned primary-key reader's complete stream is
+    /// sorted by full primary key then sequence number, and all readers collectively cover raw
+    /// mutations exactly once. Reader cardinality is independent of the number of writes. Paimon
+    /// retains `view` for the lifetime of the resulting framework reader.
     virtual Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
         const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
         const RealtimeQueryContext& context) = 0;
diff --git a/src/paimon/core/mergetree/merge_tree_writer_test.cpp b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
index 675ce3198..aa2d0c959 100644
--- a/src/paimon/core/mergetree/merge_tree_writer_test.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
@@ -21,6 +21,7 @@
 #include <cassert>
 #include <cstddef>
 #include <functional>
+#include <limits>
 #include <map>
 #include <optional>
 #include <utility>
@@ -612,6 +613,20 @@ TEST_P(MergeTreeWriterTest, TestSortedReaderFailure) {
     ASSERT_OK(merge_writer->Close());
 }
 
+TEST_P(MergeTreeWriterTest, TestRejectsExhaustedSequence) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options,
+                         CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}}));
+
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    auto path_factory = std::make_shared<DataFilePathFactory>();
+    ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr));
+
+    ASSERT_NOK_WITH_MSG(CreateMergeWriter(std::numeric_limits<int64_t>::max(), dir->Str(),
+                                          path_factory, 0, options),
+                        "sequence number has reached INT64_MAX");
+}
+
 TEST_P(MergeTreeWriterTest, TestSharedShreddingMapDataFileMetaInfo) {
     ASSERT_OK_AND_ASSIGN(CoreOptions options,
                          CoreOptions::FromMap({
diff --git a/src/paimon/core/mergetree/write_buffer.cpp b/src/paimon/core/mergetree/write_buffer.cpp
index 549975a33..3d3fdc196 100644
--- a/src/paimon/core/mergetree/write_buffer.cpp
+++ b/src/paimon/core/mergetree/write_buffer.cpp
@@ -18,6 +18,7 @@
 
 #include "paimon/core/mergetree/write_buffer.h"
 
+#include <limits>
 #include <memory>
 #include <utility>
 
@@ -39,6 +40,9 @@ Result<std::unique_ptr<WriteBuffer>> WriteBuffer::Create(
     const std::shared_ptr<MergeFunctionWrapper<KeyValue>>& merge_function_wrapper,
     const CoreOptions& options, const std::shared_ptr<IOManager>& io_manager,
     bool enable_multi_thread_spill, const std::shared_ptr<MemoryPool>& pool) {
+    if (last_sequence_number == std::numeric_limits<int64_t>::max()) {
+        return Status::Invalid("sequence number has reached INT64_MAX");
+    }
     auto value_type = arrow::struct_(value_schema->fields());
     auto in_memory_buffer = std::make_unique<InMemorySortBuffer>(
         last_sequence_number, value_type, trimmed_primary_keys, user_defined_sequence_fields,
diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp
index 492161cf8..d2c97abcf 100644
--- a/src/paimon/core/operation/key_value_file_store_write.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write.cpp
@@ -18,12 +18,13 @@
 
 #include "paimon/core/operation/key_value_file_store_write.h"
 
-#include <limits>
 #include <optional>
 #include <vector>
 
 #include "arrow/c/bridge.h"
 #include "paimon/common/data/binary_row.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/common/types/data_field.h"
 #include "paimon/core/compact/noop_compact_manager.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/io/data_file_meta.h"
@@ -35,6 +36,7 @@
 #include "paimon/core/operation/file_store_scan.h"
 #include "paimon/core/operation/key_value_file_store_scan.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
+#include "paimon/core/realtime/realtime_fields.h"
 #include "paimon/core/realtime/realtime_primary_key_writer.h"
 #include "paimon/core/schema/table_schema.h"
 #include "paimon/core/utils/file_store_path_factory.h"
@@ -124,7 +126,6 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
         std::shared_ptr<Levels> levels,
         Levels::Create(key_comparator_, restore_data_files, options_.GetNumLevels()));
     std::map<std::string, std::string> partition_map;
-    int64_t initial_max_sequence_number = restore_max_seq_number;
     std::shared_ptr<CompactManager> compact_manager;
     std::shared_ptr<RealtimeContextImpl> realtime_context_impl;
     std::optional<RealtimeStoreState> realtime_store_state;
@@ -135,19 +136,27 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
         partition_map =
             std::map<std::string, std::string>(partition_values.begin(), partition_values.end());
         PAIMON_ASSIGN_OR_RAISE(realtime_context_impl, RealtimeContextImpl::Cast(realtime_context_));
+        if (schema_->GetFieldByName(RealtimeOffsetField().Name())) {
+            return Status::Invalid("PK real-time write schema contains reserved transport field " +
+                                   RealtimeOffsetField().Name());
+        }
+        arrow::FieldVector prepared_fields = {
+            DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())
+                ->WithNullable(false),
+            DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
+                ->WithNullable(false),
+            DataField::ConvertDataFieldToArrowField(RealtimeOffsetField())->WithNullable(false)};
+        prepared_fields.insert(prepared_fields.end(), schema_->fields().begin(),
+                               schema_->fields().end());
         auto c_write_schema = std::make_unique<ArrowSchema>();
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema_, c_write_schema.get()));
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(
+            arrow::ExportSchema(*arrow::schema(std::move(prepared_fields)), c_write_schema.get()));
         PAIMON_ASSIGN_OR_RAISE(
             RealtimeStoreState store_state,
             realtime_context_impl->GetOrCreateRealtimeStore(RealtimeStoreCreateRequest{
                 std::move(c_write_schema), options_.ToMap(), pool_, partition_map, bucket,
-                PrimaryKeyRealtimeStoreCreateConfig{trimmed_primary_keys,
-                                                    restore_max_seq_number}}));
+                PrimaryKeyRealtimeStoreCreateConfig{}}));
         realtime_store_state = std::move(store_state);
-        initial_max_sequence_number = realtime_store_state->initial_max_sequence_number.value();
-        if (initial_max_sequence_number == std::numeric_limits<int64_t>::max()) {
-            return Status::Invalid("PK sequence number has reached INT64_MAX");
-        }
         compact_manager = std::make_shared<NoopCompactManager>();
     } else {
         auto compact_strategy = compact_manager_factory_->CreateCompactStrategy();
@@ -159,15 +168,16 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
     PAIMON_ASSIGN_OR_RAISE(
         std::shared_ptr<MergeTreeWriter> writer,
         MergeTreeWriter::Create(
-            initial_max_sequence_number, trimmed_primary_keys, data_file_path_factory,
-            key_comparator_, user_defined_seq_comparator_, merge_function_wrapper_,
-            table_schema_->Id(), schema_, options_, compact_manager,
-            realtime_context_ ? nullptr : io_manager_, enable_multi_thread_spill_, pool_));
+            restore_max_seq_number, trimmed_primary_keys, data_file_path_factory, key_comparator_,
+            user_defined_seq_comparator_, merge_function_wrapper_, table_schema_->Id(), schema_,
+            options_, compact_manager, realtime_context_ ? nullptr : io_manager_,
+            enable_multi_thread_spill_, pool_));
     if (!realtime_context_) {
         return std::shared_ptr<BatchWriter>(std::move(writer));
     }
-    return RealtimePrimaryKeyWriter::Create(partition_map, bucket, schema_, realtime_context_impl,
-                                            writer, pool_, realtime_store_state.value());
+    return RealtimePrimaryKeyWriter::Create(schema_, trimmed_primary_keys, key_comparator_,
+                                            realtime_store_state.value(), restore_max_seq_number,
+                                            writer, pool_);
 }
 
 Status KeyValueFileStoreWrite::RefreshCommittedSnapshot(int64_t snapshot_id) {
diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp
index 45462ea6e..cbd2189fc 100644
--- a/src/paimon/core/operation/key_value_file_store_write_test.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp
@@ -19,9 +19,13 @@
 #include "paimon/core/operation/key_value_file_store_write.h"
 
 #include <cstddef>
+#include <limits>
 #include <map>
 #include <memory>
+#include <new>
+#include <optional>
 #include <string>
+#include <tuple>
 #include <utility>
 #include <vector>
 
@@ -44,6 +48,7 @@
 #include "paimon/common/utils/path_util.h"
 #include "paimon/core/io/data_file_meta.h"
 #include "paimon/core/operation/restore_files.h"
+#include "paimon/core/realtime/realtime_context_impl.h"
 #include "paimon/core/stats/simple_stats.h"
 #include "paimon/core/table/sink/commit_message_impl.h"
 #include "paimon/file_store_commit.h"
@@ -52,7 +57,9 @@
 #include "paimon/format/file_format_factory.h"
 #include "paimon/format/reader_builder.h"
 #include "paimon/fs/local/local_file_system.h"
+#include "paimon/memory/memory_pool.h"
 #include "paimon/reader/file_batch_reader.h"
+#include "paimon/realtime/arrow_realtime_store_factory.h"
 #include "paimon/realtime/realtime_context.h"
 #include "paimon/record_batch.h"
 #include "paimon/status.h"
@@ -61,6 +68,113 @@
 #include "paimon/write_context.h"
 
 namespace paimon::test {
+namespace {
+
+class TestingMemoryPool final : public MemoryPool {
+ public:
+    void* Malloc(uint64_t size, uint64_t alignment) override {
+        ++allocation_count;
+        if (reject_allocations) {
+            throw std::bad_alloc();
+        }
+        return delegate_->Malloc(size, alignment);
+    }
+
+    void* Realloc(void* pointer, size_t old_size, size_t new_size, uint64_t alignment) override {
+        ++allocation_count;
+        if (reject_allocations) {
+            throw std::bad_alloc();
+        }
+        return delegate_->Realloc(pointer, old_size, new_size, alignment);
+    }
+
+    void Free(void* pointer, uint64_t size) override {
+        delegate_->Free(pointer, size);
+    }
+
+    void Free(void* pointer, uint64_t size, uint64_t alignment) override {
+        delegate_->Free(pointer, size, alignment);
+    }
+
+    uint64_t CurrentUsage() const override {
+        return delegate_->CurrentUsage();
+    }
+
+    uint64_t MaxMemoryUsage() const override {
+        return delegate_->MaxMemoryUsage();
+    }
+
+    bool reject_allocations = false;
+    int64_t allocation_count = 0;
+
+ private:
+    std::unique_ptr<MemoryPool> delegate_ = GetMemoryPool();
+};
+
+class FailOnceRealtimeStore final : public RealtimeStore {
+ public:
+    FailOnceRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
+                          const std::shared_ptr<bool>& fail_next_write)
+        : delegate_(delegate), fail_next_write_(fail_next_write) {}
+
+    Status Write(RealtimeWriteBatch&& batch) override {
+        if (*fail_next_write_) {
+            *fail_next_write_ = false;
+            return Status::Invalid("injected real-time store write failure");
+        }
+        return delegate_->Write(std::move(batch));
+    }
+
+    Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>> SealForCommit() override {
+        return delegate_->SealForCommit();
+    }
+
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
+        const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
+        return delegate_->CreateCommitReaders(segment);
+    }
+
+    Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
+        return delegate_->AcquireReadView();
+    }
+
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
+        const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
+        const RealtimeQueryContext& context) override {
+        return delegate_->CreateQueryReaders(view, offset_begin, context);
+    }
+
+    Status AdvanceCommittedOffset(int64_t committed_offset) override {
+        return delegate_->AdvanceCommittedOffset(committed_offset);
+    }
+
+    uint64_t GetMemoryUsage() const override {
+        return delegate_->GetMemoryUsage();
+    }
+
+ private:
+    std::shared_ptr<RealtimeStore> delegate_;
+    std::shared_ptr<bool> fail_next_write_;
+};
+
+class FailOnceRealtimeStoreFactory final : public RealtimeStoreFactory {
+ public:
+    explicit FailOnceRealtimeStoreFactory(const std::shared_ptr<bool>& fail_next_write)
+        : fail_next_write_(fail_next_write) {}
+
+    Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> delegate,
+                               delegate_.Create(std::move(request)));
+        return std::shared_ptr<RealtimeStore>(
+            std::make_shared<FailOnceRealtimeStore>(delegate, fail_next_write_));
+    }
+
+ private:
+    ArrowRealtimeStoreFactory delegate_;
+    std::shared_ptr<bool> fail_next_write_;
+};
+
+}
 
 class KeyValueFileStoreWriteTest : public ::testing::Test {
  protected:
@@ -128,14 +242,15 @@ class KeyValueFileStoreWriteTest : public ::testing::Test {
                                        /*ignore_if_exists=*/false));
     }
 
-    std::unique_ptr<RecordBatch> MakeBatch(const std::shared_ptr<arrow::Schema>& schema,
-                                           const std::string& json) const {
+    std::unique_ptr<RecordBatch> MakeBatch(
+        const std::shared_ptr<arrow::Schema>& schema, const std::string& json,
+        const std::vector<RecordBatch::RowKind>& row_kinds = {}) const {
         auto struct_type = arrow::struct_(schema->fields());
         auto array = arrow::ipc::internal::json::ArrayFromJSON(struct_type, json).ValueOrDie();
         ::ArrowArray arrow_array;
         EXPECT_TRUE(arrow::ExportArray(*array, &arrow_array).ok());
         RecordBatchBuilder batch_builder(&arrow_array);
-        return batch_builder.SetBucket(0).Finish().value();
+        return batch_builder.SetRowKinds(row_kinds).SetBucket(0).Finish().value();
     }
 
     std::vector<std::shared_ptr<CommitMessage>> WriteAndPrepare(
@@ -194,6 +309,58 @@ class KeyValueFileStoreWriteTest : public ::testing::Test {
         EXPECT_NE(nullptr, metadata);
         return MapSharedShreddingUtils::DeserializeMetadata(metadata->Copy()).value();
     }
+
+    Result<std::vector<std::tuple<int8_t, int64_t, std::string, int64_t, int64_t>>>
+    ReadPreparedRows(const std::shared_ptr<RealtimeContext>& realtime_context) const {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> context,
+                               RealtimeContextImpl::Cast(realtime_context));
+        PAIMON_ASSIGN_OR_RAISE(std::vector<RealtimePartitionBucketView> views,
+                               context->AcquireReadViews());
+        if (views.size() != 1) {
+            return Status::Invalid("expected exactly one real-time store");
+        }
+        RealtimeQueryContext query_context{nullptr, nullptr, false};
+        PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
+                               views[0].store->CreateQueryReaders(
+                                   views[0].read_view, 0, query_context));
+        std::vector<std::tuple<int8_t, int64_t, std::string, int64_t, int64_t>> rows;
+        for (const std::unique_ptr<BatchReader>& reader : readers) {
+            while (true) {
+                PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader->NextBatch());
+                if (BatchReader::IsEofBatch(batch)) {
+                    break;
+                }
+                PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+                    std::shared_ptr<arrow::Array> array,
+                    arrow::ImportArray(batch.first.get(), batch.second.get()));
+                std::shared_ptr<arrow::StructArray> values =
+                    std::dynamic_pointer_cast<arrow::StructArray>(array);
+                if (!values || values->num_fields() != 5) {
+                    return Status::Invalid("unexpected prepared real-time batch");
+                }
+                std::shared_ptr<arrow::Int8Array> row_kinds =
+                    std::dynamic_pointer_cast<arrow::Int8Array>(values->field(0));
+                std::shared_ptr<arrow::Int64Array> sequences =
+                    std::dynamic_pointer_cast<arrow::Int64Array>(values->field(1));
+                std::shared_ptr<arrow::Int64Array> offsets =
+                    std::dynamic_pointer_cast<arrow::Int64Array>(values->field(2));
+                std::shared_ptr<arrow::Int64Array> ids =
+                    std::dynamic_pointer_cast<arrow::Int64Array>(values->field(3));
+                std::shared_ptr<arrow::StringArray> payloads =
+                    std::dynamic_pointer_cast<arrow::StringArray>(values->field(4));
+                if (!row_kinds || !sequences || !offsets || !ids || !payloads) {
+                    return Status::Invalid("unexpected prepared real-time column type");
+                }
+                for (int64_t row = 0; row < values->length(); ++row) {
+                    rows.emplace_back(row_kinds->Value(row), ids->Value(row),
+                                      payloads->GetString(row), sequences->Value(row),
+                                      offsets->Value(row));
+                }
+            }
+            reader->Close();
+        }
+        return rows;
+    }
 };
 
 TEST_F(KeyValueFileStoreWriteTest, TestWriteWithInvalidBatch) {
@@ -310,7 +477,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeWrite) {
         {Options::WRITE_BUFFER_SIZE, "1"},
     };
     const std::shared_ptr<arrow::Schema> schema = arrow::schema({
-        arrow::field("id", arrow::int64(), /*nullable=*/false),
+        arrow::field("id", arrow::int64(), false),
         arrow::field("value", arrow::utf8()),
     });
     std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
@@ -329,13 +496,23 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeWrite) {
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                          FileStoreWrite::Create(std::move(write_context)));
 
-    ASSERT_OK(writer->Write(MakeBatch(schema, R"([
+    std::unique_ptr<RecordBatch> batch =
+        MakeBatch(schema, R"([
         [1, "old"],
         [2, "two"],
         [1, "new"]
-    ])")));
+    ])",
+                  {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::DELETE,
+                   RecordBatch::RowKind::UPDATE_AFTER});
+    ASSERT_OK(writer->Write(std::move(batch)));
+    using PreparedRow = std::tuple<int8_t, int64_t, std::string, int64_t, int64_t>;
+    ASSERT_OK_AND_ASSIGN(std::vector<PreparedRow> prepared_rows,
+                         ReadPreparedRows(realtime_context));
+    ASSERT_EQ(
+        (std::vector<PreparedRow>{{0, 1, "old", 0, 0}, {2, 1, "new", 2, 2}, {3, 2, "two", 1, 1}}),
+        prepared_rows);
     ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progresses,
-                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+                         writer->PrepareCommitWithProgress(0));
     ASSERT_EQ(1, progresses.size());
     ASSERT_EQ(OffsetRange(0, 3), progresses[0].offset_range);
     std::shared_ptr<CommitMessageImpl> commit_message =
@@ -351,6 +528,189 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeWrite) {
     ASSERT_OK(writer->Close());
 }
 
+TEST_F(KeyValueFileStoreWriteTest, TestRealtimeOffsetCollision) {
+    const std::map<std::string, std::string> options = {{Options::BUCKET, "1"}};
+    const std::shared_ptr<arrow::Schema> schema = arrow::schema({
+        arrow::field("id", arrow::int64(), false),
+        arrow::field("_REALTIME_OFFSET", arrow::int64()),
+    });
+    std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    CreateTable(dir->Str(), schema, options);
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    WriteContextBuilder builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), "test");
+    builder.SetOptions(options).WithStreamingMode(true).WithRealtimeContext(realtime_context);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> write_context, builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         FileStoreWrite::Create(std::move(write_context)));
+
+    ASSERT_NOK_WITH_MSG(writer->Write(MakeBatch(schema, R"([[1, 10]])")),
+                        "PK real-time write schema contains reserved transport field");
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(KeyValueFileStoreWriteTest, TestWriteFailureKeepsCursors) {
+    const std::map<std::string, std::string> options = {
+        {Options::BUCKET, "1"},
+        {Options::WRITE_BUFFER_SIZE, "1"},
+    };
+    const std::shared_ptr<arrow::Schema> schema = arrow::schema({
+        arrow::field("id", arrow::int64(), false),
+        arrow::field("value", arrow::utf8()),
+    });
+    std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    CreateTable(dir->Str(), schema, options);
+    const std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar");
+
+    auto fail_next_write = std::make_shared<bool>(true);
+    auto factory = std::make_shared<FailOnceRealtimeStoreFactory>(fail_next_write);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create(factory));
+    WriteContextBuilder builder(table_path, "test");
+    builder.SetOptions(options).WithStreamingMode(true).WithRealtimeContext(realtime_context);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> write_context, builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         FileStoreWrite::Create(std::move(write_context)));
+
+    ASSERT_NOK_WITH_MSG(writer->Write(MakeBatch(schema, R"([[9, "rejected"]])")),
+                        "injected real-time store write failure");
+    ASSERT_OK(writer->Write(MakeBatch(schema, R"([[1, "kept"]])")));
+    using PreparedRow = std::tuple<int8_t, int64_t, std::string, int64_t, int64_t>;
+    ASSERT_OK_AND_ASSIGN(std::vector<PreparedRow> prepared_rows,
+                         ReadPreparedRows(realtime_context));
+    ASSERT_EQ((std::vector<PreparedRow>{{0, 1, "kept", 0, 0}}), prepared_rows);
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) {
+    const std::map<std::string, std::string> options = {{Options::BUCKET, "1"}};
+    const std::shared_ptr<arrow::Schema> schema = arrow::schema({
+        arrow::field("id", arrow::int64(), false),
+        arrow::field("value", arrow::utf8()),
+    });
+    std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    CreateTable(dir->Str(), schema, options);
+    const std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar");
+
+    std::shared_ptr<TestingMemoryPool> pool = std::make_shared<TestingMemoryPool>();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    WriteContextBuilder builder(table_path, "test");
+    builder.SetOptions(options)
+        .WithStreamingMode(true)
+        .WithRealtimeContext(realtime_context)
+        .WithMemoryPool(pool);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> write_context, builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         FileStoreWrite::Create(std::move(write_context)));
+
+    const int64_t allocations_before_write = pool->allocation_count;
+    ASSERT_OK(writer->Write(MakeBatch(schema, R"([[1, "one"]])")));
+    ASSERT_GT(pool->allocation_count, allocations_before_write);
+    ASSERT_OK(writer->Close());
+    writer.reset();
+    using PreparedRow = std::tuple<int8_t, int64_t, std::string, int64_t, int64_t>;
+    ASSERT_OK_AND_ASSIGN(std::vector<PreparedRow> retained_rows,
+                         ReadPreparedRows(realtime_context));
+    ASSERT_EQ((std::vector<PreparedRow>{{0, 1, "one", 0, 0}}), retained_rows);
+
+    std::shared_ptr<TestingMemoryPool> rejecting_pool = std::make_shared<TestingMemoryPool>();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> rejecting_context,
+                         RealtimeContext::Create());
+    WriteContextBuilder rejecting_builder(table_path, "rejecting");
+    rejecting_builder.SetOptions(options)
+        .WithStreamingMode(true)
+        .WithRealtimeContext(rejecting_context)
+        .WithMemoryPool(rejecting_pool);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> rejecting_write_context,
+                         rejecting_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> rejecting_writer,
+                         FileStoreWrite::Create(std::move(rejecting_write_context)));
+    ASSERT_OK(rejecting_writer->Write(MakeBatch(schema, "[]")));
+    const int64_t rejecting_allocations_before_write = rejecting_pool->allocation_count;
+    rejecting_pool->reject_allocations = true;
+    ASSERT_NOK_WITH_MSG(rejecting_writer->Write(MakeBatch(schema, R"([[2, "two"]])")),
+                        "Out of memory");
+    ASSERT_GT(rejecting_pool->allocation_count, rejecting_allocations_before_write);
+    ASSERT_OK_AND_ASSIGN(std::vector<PreparedRow> rejected_rows,
+                         ReadPreparedRows(rejecting_context));
+    ASSERT_TRUE(rejected_rows.empty());
+    ASSERT_OK(rejecting_writer->Close());
+}
+
+TEST_F(KeyValueFileStoreWriteTest, TestRealtimeLimits) {
+    const int64_t max = std::numeric_limits<int64_t>::max();
+    const std::map<std::string, std::string> options = {{Options::BUCKET, "1"}};
+    const std::shared_ptr<arrow::Schema> schema = arrow::schema({
+        arrow::field("id", arrow::int64(), false),
+        arrow::field("value", arrow::utf8()),
+    });
+    std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    CreateTable(dir->Str(), schema, options);
+    const std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar");
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> initial_context,
+                         RealtimeContext::Create());
+    WriteContextBuilder initial_builder(table_path, "initial");
+    initial_builder.SetOptions(options).WithStreamingMode(true).WithRealtimeContext(
+        initial_context);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> initial_write_context,
+                         initial_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> initial_writer,
+                         FileStoreWrite::Create(std::move(initial_write_context)));
+    ASSERT_OK(initial_writer->Write(MakeBatch(schema, R"([[0, "initial"]])")));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> initial_progress,
+                         initial_writer->PrepareCommitWithProgress(0));
+    ASSERT_EQ(1, initial_progress.size());
+    std::shared_ptr<CommitMessageImpl> initial_message =
+        std::dynamic_pointer_cast<CommitMessageImpl>(initial_progress[0].commit_message);
+    ASSERT_NE(nullptr, initial_message);
+    ASSERT_EQ(1, initial_message->GetNewFilesIncrement().NewFiles().size());
+    initial_message->GetNewFilesIncrement().NewFiles()[0]->AssignSequenceNumber(max - 2, max - 2);
+    initial_progress[0].offset_range = OffsetRange(0, max - 1);
+
+    CommitContextBuilder commit_builder(table_path, "initial");
+    commit_builder.SetOptions(options);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context, commit_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> committer,
+                         FileStoreCommit::Create(std::move(commit_context)));
+    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id,
+                         committer->CommitWithProgress(initial_progress, 0, std::nullopt));
+    ASSERT_OK(initial_writer->Close());
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    WriteContextBuilder builder(table_path, "boundary");
+    builder.SetOptions(options).WithStreamingMode(true).WithRealtimeContext(realtime_context);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> write_context, builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         FileStoreWrite::Create(std::move(write_context)));
+    ASSERT_OK(writer->Write(MakeBatch(schema, R"([[1, "legal"]])")));
+    using PreparedRow = std::tuple<int8_t, int64_t, std::string, int64_t, int64_t>;
+    ASSERT_OK_AND_ASSIGN(std::vector<PreparedRow> prepared_rows,
+                         ReadPreparedRows(realtime_context));
+    ASSERT_EQ((std::vector<PreparedRow>{{0, 1, "legal", max - 1, max - 1}}), prepared_rows);
+
+    ASSERT_NOK_WITH_MSG(writer->Write(MakeBatch(schema, R"([[2, "overflow"]])")),
+                        "real-time offset range exceeds INT64_MAX");
+    ASSERT_OK_AND_ASSIGN(prepared_rows, ReadPreparedRows(realtime_context));
+    ASSERT_EQ((std::vector<PreparedRow>{{0, 1, "legal", max - 1, max - 1}}), prepared_rows);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context_impl,
+                         RealtimeContextImpl::Cast(realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimePartitionBucketView> views,
+                         context_impl->AcquireReadViews());
+    ASSERT_EQ(1, views.size());
+    ASSERT_EQ(std::optional<OffsetRange>(OffsetRange(max - 1, max)),
+              views[0].read_view->GetOffsetRange());
+    ASSERT_OK(writer->Close());
+    ASSERT_GE(snapshot_id, 1);
+}
+
 TEST_F(KeyValueFileStoreWriteTest,
        TestPrepareCommitShouldSucceedWhenDefaultCompactRewriterPathEnabled) {
     ASSERT_OK_AND_ASSIGN(
diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp
index 8d8367e39..2f64f6df8 100644
--- a/src/paimon/core/operation/merge_file_split_read.cpp
+++ b/src/paimon/core/operation/merge_file_split_read.cpp
@@ -81,8 +81,6 @@ class MergeFunctionWrapper;
 
 namespace {
 
-/// Concatenates merge readers whose key ranges are ordered and non-overlapping, preserving one
-/// projection pipeline without merging independent disk-only components.
 class ConcatNonOverlappingMergeReaders final : public SortMergeReader {
  public:
     explicit ConcatNonOverlappingMergeReaders(
@@ -117,7 +115,7 @@ class ConcatNonOverlappingMergeReaders final : public SortMergeReader {
     size_t current_ = 0;
 };
 
-}  // namespace
+}
 
 class MergeFileSplitRead::RealtimeReaderBuilder {
  public:
@@ -219,8 +217,8 @@ class MergeFileSplitRead::RealtimeReaderBuilder {
         inputs_.reserve(inputs_.size() + additional_readers.size());
         for (AdditionalKeyValueReader& additional : additional_readers) {
             has_unknown_range_ |= !additional.min_key || !additional.max_key;
-            inputs_.push_back(RangeInput{additional.min_key, additional.max_key,
-                                         /*disk_runs=*/{}, std::move(additional.reader)});
+            inputs_.push_back(RangeInput{additional.min_key, additional.max_key, {},
+                                         std::move(additional.reader)});
         }
     }
 
@@ -310,7 +308,7 @@ class MergeFileSplitRead::RealtimeReaderBuilder {
                         component.front().disk_runs, first_split_->Partition(), dv_factory_,
                         component.front().disk_runs.size() == 1 ? owner_->context_->GetPredicate()
                                                                 : owner_->predicate_for_keys_,
-                        data_file_path_factory_, /*drop_delete=*/false));
+                        data_file_path_factory_, false));
                 component_readers.push_back(std::move(disk_component));
                 continue;
             }
diff --git a/src/paimon/core/operation/merge_file_split_read.h b/src/paimon/core/operation/merge_file_split_read.h
index 0824254b7..85b5b2a28 100644
--- a/src/paimon/core/operation/merge_file_split_read.h
+++ b/src/paimon/core/operation/merge_file_split_read.h
@@ -127,7 +127,6 @@ class MergeFileSplitRead : public AbstractSplitRead {
         return key_schema_;
     }
 
-    /// Merges ordinary disk splits with generic additional sorted KeyValue readers.
     Result<std::unique_ptr<BatchReader>> CreateRealtimeReader(
         const std::vector<std::shared_ptr<Split>>& disk_splits,
         std::vector<AdditionalKeyValueReader>&& additional_readers);
diff --git a/src/paimon/core/realtime/arrow_realtime_store_factory.cpp b/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
index e6e22edfd..4cfdb4c3d 100644
--- a/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
+++ b/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
@@ -21,14 +21,9 @@
 
 #include "arrow/c/bridge.h"
 #include "arrow/c/helpers.h"
-#include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
-#include "paimon/common/utils/fields_comparator.h"
 #include "paimon/common/utils/scope_guard.h"
-#include "paimon/core/core_options.h"
-#include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
-#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
 #include "paimon/core/realtime/arrow_realtime_store.h"
 #include "paimon/core/realtime/primary_key_realtime_store.h"
 #include "paimon/macros.h"
@@ -55,31 +50,8 @@ Result<std::shared_ptr<RealtimeStore>> ArrowRealtimeStoreFactory::Create(
                                                     request.memory_pool, arrow_pool);
     }
 
-    const PrimaryKeyRealtimeStoreCreateConfig& primary_key_config =
-        std::get<PrimaryKeyRealtimeStoreCreateConfig>(request.mode_config);
-    std::vector<DataField> key_fields;
-    key_fields.reserve(primary_key_config.primary_keys.size());
-    for (const std::string& primary_key : primary_key_config.primary_keys) {
-        const int32_t field_index = imported_schema->GetFieldIndex(primary_key);
-        if (field_index < 0) {
-            return Status::Invalid("primary key ", primary_key, " is missing from write schema");
-        }
-        key_fields.emplace_back(field_index, imported_schema->field(field_index));
-    }
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FieldsComparator> key_comparator,
-                           FieldsComparator::Create(key_fields, /*is_ascending_order=*/true));
-    auto merge_function_wrapper_factory = []() {
-        auto merge_function = std::make_unique<DeduplicateMergeFunction>(
-            /*ignore_delete=*/false);
-        return std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge_function));
-    };
-    PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(request.options));
-    PAIMON_ASSIGN_OR_RAISE(
-        std::shared_ptr<PrimaryKeyRealtimeStore> store,
-        PrimaryKeyRealtimeStore::Create(imported_schema, primary_key_config.primary_keys,
-                                        key_comparator, merge_function_wrapper_factory,
-                                        primary_key_config.restore_max_sequence_number,
-                                        core_options.GetReadBatchSize(), request.memory_pool));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                           PrimaryKeyRealtimeStore::Create(imported_schema, request.memory_pool));
     return std::shared_ptr<RealtimeStore>(std::move(store));
 }
 
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index 8f51c1b1e..0d6de9f5f 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -9,41 +9,24 @@
  *
  *   http://www.apache.org/licenses/LICENSE-2.0
  *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 #include "paimon/core/realtime/primary_key_realtime_store.h"
 
-#include <algorithm>
-#include <limits>
 #include <mutex>
 #include <utility>
 
 #include "arrow/api.h"
 #include "arrow/c/bridge.h"
-#include "paimon/common/data/binary_row_writer.h"
-#include "paimon/common/data/columnar/columnar_row_ref.h"
 #include "paimon/common/metrics/metrics_impl.h"
-#include "paimon/common/table/special_fields.h"
-#include "paimon/common/types/data_field.h"
-#include "paimon/common/types/row_kind.h"
-#include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
-#include "paimon/common/utils/fields_comparator.h"
 #include "paimon/core/core_options.h"
-#include "paimon/core/io/key_value_in_memory_record_reader.h"
-#include "paimon/core/io/key_value_projection_consumer.h"
-#include "paimon/core/io/key_value_projection_reader.h"
-#include "paimon/core/io/merged_key_value_record_reader.h"
-#include "paimon/core/key_value.h"
-#include "paimon/core/mergetree/compact/sort_merge_reader_with_loser_tree.h"
-#include "paimon/core/utils/nested_projection_utils.h"
 #include "paimon/macros.h"
 
 namespace paimon {
@@ -83,562 +66,255 @@ Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options) {
 namespace {
 
 uint64_t GetArrayMemoryUsage(const std::shared_ptr<arrow::ArrayData>& data) {
-    uint64_t result = 0;
+    uint64_t total = 0;
     for (const std::shared_ptr<arrow::Buffer>& buffer : data->buffers) {
         if (buffer) {
-            result += static_cast<uint64_t>(buffer->size());
+            total += static_cast<uint64_t>(buffer->size());
         }
     }
     for (const std::shared_ptr<arrow::ArrayData>& child : data->child_data) {
-        result += GetArrayMemoryUsage(child);
+        total += GetArrayMemoryUsage(child);
     }
     if (data->dictionary) {
-        result += GetArrayMemoryUsage(data->dictionary);
+        total += GetArrayMemoryUsage(data->dictionary);
     }
-    return result;
-}
-
-int32_t FindPkQueryFieldIndex(const std::shared_ptr<arrow::Schema>& write_schema,
-                              const std::shared_ptr<arrow::Field>& read_field) {
-    Result<int32_t> read_id = NestedProjectionUtils::GetPaimonFieldId(read_field);
-    if (read_id.ok()) {
-        Result<std::shared_ptr<arrow::Field>> write_field =
-            NestedProjectionUtils::FindFieldByPaimonId(arrow::struct_(write_schema->fields()),
-                                                       read_id.value());
-        if (write_field.ok()) {
-            return write_schema->GetFieldIndex(write_field.value()->name());
-        }
-    }
-
-    const int32_t name_index = write_schema->GetFieldIndex(read_field->name());
-    if (name_index < 0) {
-        return -1;
-    }
-    Result<int32_t> write_id =
-        NestedProjectionUtils::GetPaimonFieldId(write_schema->field(name_index));
-    if (read_id.ok() && write_id.ok() && read_id.value() != write_id.value()) {
-        return -1;
-    }
-    return name_index;
+    return total;
 }
 
 struct StoredBatch {
     std::shared_ptr<arrow::StructArray> data;
-    std::vector<RecordBatch::RowKind> row_kinds;
     OffsetRange offset_range;
-    int64_t first_sequence_number;
     uint64_t memory_usage;
 };
-using BatchGroup = std::vector<std::shared_ptr<const StoredBatch>>;
 
 class Segment final : public RealtimeSegmentHandle {
  public:
-    Segment(const OffsetRange& offset_range,
-            std::vector<std::shared_ptr<const StoredBatch>>&& batches)
-        : offset_range_(offset_range), batches_(std::move(batches)) {}
+    Segment(const OffsetRange& range, std::vector<StoredBatch>&& batches)
+        : range_(range), batches_(std::move(batches)) {}
 
     OffsetRange GetOffsetRange() const override {
-        return offset_range_;
+        return range_;
     }
-
-    const std::vector<std::shared_ptr<const StoredBatch>>& Batches() const {
+    const std::vector<StoredBatch>& Batches() const {
         return batches_;
     }
 
-    uint64_t GetMemoryUsage() const {
-        uint64_t result = 0;
-        for (const std::shared_ptr<const StoredBatch>& batch : batches_) {
-            result += batch->memory_usage;
-        }
-        return result;
-    }
-
  private:
-    OffsetRange offset_range_;
-    std::vector<std::shared_ptr<const StoredBatch>> batches_;
+    OffsetRange range_;
+    std::vector<StoredBatch> batches_;
 };
 
-class PrimaryKeyRealtimeReadView final : public RealtimeReadView {
+class ReadView final : public RealtimeReadView {
  public:
-    explicit PrimaryKeyRealtimeReadView(std::vector<BatchGroup>&& groups)
-        : groups_(std::move(groups)) {
-        if (!groups_.empty()) {
-            offset_range_ = OffsetRange(groups_.front().front()->offset_range.begin,
-                                        groups_.back().back()->offset_range.end);
+    explicit ReadView(std::vector<std::shared_ptr<Segment>>&& segments)
+        : segments_(std::move(segments)) {
+        if (!segments_.empty()) {
+            range_ = OffsetRange(segments_.front()->GetOffsetRange().begin,
+                                 segments_.back()->GetOffsetRange().end);
         }
     }
 
     std::optional<OffsetRange> GetOffsetRange() const override {
-        return offset_range_;
+        return range_;
     }
-
-    const std::vector<BatchGroup>& Groups() const {
-        return groups_;
+    const std::vector<std::shared_ptr<Segment>>& Segments() const {
+        return segments_;
     }
 
  private:
-    std::vector<BatchGroup> groups_;
-    std::optional<OffsetRange> offset_range_;
+    std::vector<std::shared_ptr<Segment>> segments_;
+    std::optional<OffsetRange> range_;
 };
 
-class CommitBatchReader final : public BatchReader {
+class RawBatchReader final : public BatchReader {
  public:
-    CommitBatchReader(const std::shared_ptr<Segment>& segment,
-                      const std::shared_ptr<arrow::MemoryPool>& arrow_pool)
-        : segment_(segment), arrow_pool_(arrow_pool), metrics_(std::make_shared<MetricsImpl>()) {}
+    RawBatchReader(std::vector<StoredBatch> batches)
+        : batches_(std::move(batches)), metrics_(std::make_shared<MetricsImpl>()) {}
 
     Result<ReadBatch> NextBatch() override {
-        if (!segment_ || next_batch_ >= static_cast<int32_t>(segment_->Batches().size())) {
+        if (next_ == batches_.size()) {
             return MakeEofBatch();
         }
-        const std::shared_ptr<const StoredBatch>& stored = segment_->Batches()[next_batch_++];
-        const int64_t row_count = stored->data->length();
-        arrow::Int8Builder row_kind_builder(arrow_pool_.get());
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(row_kind_builder.Reserve(row_count));
-        if (stored->row_kinds.empty()) {
-            for (int64_t i = 0; i < row_count; ++i) {
-                row_kind_builder.UnsafeAppend(static_cast<int8_t>(RecordBatch::RowKind::INSERT));
-            }
-        } else {
-            for (RecordBatch::RowKind row_kind : stored->row_kinds) {
-                row_kind_builder.UnsafeAppend(static_cast<int8_t>(row_kind));
-            }
-        }
-        std::shared_ptr<arrow::Array> row_kind_array;
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(row_kind_builder.Finish(&row_kind_array));
-        arrow::ArrayVector arrays = {std::move(row_kind_array)};
-        arrays.insert(arrays.end(), stored->data->fields().begin(), stored->data->fields().end());
-        arrow::FieldVector fields = {
-            DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())};
-        const arrow::FieldVector& value_fields = stored->data->struct_type()->fields();
-        fields.insert(fields.end(), value_fields.begin(), value_fields.end());
-        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::StructArray> output,
-                                          arrow::StructArray::Make(arrays, fields));
-        auto c_array = std::make_unique<ArrowArray>();
-        auto c_schema = std::make_unique<ArrowSchema>();
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*output, c_array.get(), c_schema.get()));
-        return ReadBatch(std::move(c_array), std::move(c_schema));
+        const std::shared_ptr<arrow::StructArray>& batch = batches_[next_++].data;
+        auto array = std::make_unique<ArrowArray>();
+        auto schema = std::make_unique<ArrowSchema>();
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*batch, array.get(), schema.get()));
+        return ReadBatch(std::move(array), std::move(schema));
     }
 
     std::shared_ptr<Metrics> GetReaderMetrics() const override {
         return metrics_;
     }
-
     void Close() override {
-        segment_.reset();
+        batches_.clear();
     }
 
  private:
-    std::shared_ptr<Segment> segment_;
-    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
+    std::vector<StoredBatch> batches_;
+    size_t next_ = 0;
     std::shared_ptr<Metrics> metrics_;
-    int32_t next_batch_ = 0;
-};
-
-class KeyRangeBatchReader final : public BatchReader, public PrimaryKeyRangeProvider {
- public:
-    KeyRangeBatchReader(std::unique_ptr<BatchReader>&& reader,
-                        const std::shared_ptr<InternalRow>& min_key,
-                        const std::shared_ptr<InternalRow>& max_key)
-        : reader_(std::move(reader)), min_key_(min_key), max_key_(max_key) {}
-
-    Result<ReadBatch> NextBatch() override {
-        return reader_->NextBatch();
-    }
-
-    std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return reader_->GetReaderMetrics();
-    }
-
-    void Close() override {
-        reader_->Close();
-    }
-
-    std::shared_ptr<InternalRow> GetMinKey() const override {
-        return min_key_;
-    }
-
-    std::shared_ptr<InternalRow> GetMaxKey() const override {
-        return max_key_;
-    }
-
- private:
-    std::unique_ptr<BatchReader> reader_;
-    std::shared_ptr<InternalRow> min_key_;
-    std::shared_ptr<InternalRow> max_key_;
 };
 
 }  // namespace
 
 class PrimaryKeyRealtimeStore::Impl {
  public:
-    Impl(const std::shared_ptr<arrow::Schema>& write_schema, std::vector<std::string> primary_keys,
-         const std::shared_ptr<FieldsComparator>& key_comparator,
-         const std::function<std::shared_ptr<MergeFunctionWrapper<KeyValue>>()>&
-             merge_function_wrapper_factory,
-         int64_t next_sequence_number, int32_t read_batch_size,
-         const std::shared_ptr<MemoryPool>& memory_pool)
-        : write_schema_(write_schema),
-          primary_keys_(std::move(primary_keys)),
-          key_comparator_(key_comparator),
-          merge_function_wrapper_factory_(merge_function_wrapper_factory),
-          next_sequence_number_(next_sequence_number),
-          read_batch_size_(read_batch_size),
-          memory_pool_(memory_pool),
-          arrow_pool_(GetArrowPool(memory_pool)) {}
-
-    Result<std::shared_ptr<InternalRow>> CopyKey(const InternalRow& key) const {
-        auto result = std::make_shared<BinaryRow>(static_cast<int32_t>(primary_keys_.size()));
-        BinaryRowWriter writer(result.get(), /*initial_size=*/128, memory_pool_.get());
-        writer.Reset();
-        for (int32_t index = 0; index < static_cast<int32_t>(primary_keys_.size()); ++index) {
-            std::shared_ptr<arrow::Field> field =
-                write_schema_->GetFieldByName(primary_keys_[index]);
-            PAIMON_ASSIGN_OR_RAISE(InternalRow::FieldGetterFunc getter,
-                                   InternalRow::CreateFieldGetter(index, field->type(),
-                                                                  /*use_view=*/true));
-            PAIMON_ASSIGN_OR_RAISE(BinaryRowWriter::FieldSetterFunc setter,
-                                   BinaryRowWriter::CreateFieldSetter(index, field->type()));
-            setter(getter(key), &writer);
-        }
-        writer.Complete();
-        return std::static_pointer_cast<InternalRow>(result);
-    }
-
-    Result<std::pair<std::shared_ptr<InternalRow>, std::shared_ptr<InternalRow>>> GetKeyRange(
-        const std::shared_ptr<arrow::StructArray>& values) const {
-        arrow::ArrayVector key_arrays;
-        key_arrays.reserve(primary_keys_.size());
-        for (const std::string& primary_key : primary_keys_) {
-            std::shared_ptr<arrow::Array> key_array = values->GetFieldByName(primary_key);
-            if (!key_array) {
-                return Status::Invalid("primary key is missing from PK query batch: ", primary_key);
-            }
-            key_arrays.push_back(std::move(key_array));
-        }
-        auto context = std::make_shared<ColumnarBatchContext>(key_arrays, memory_pool_);
-        int64_t min_row = 0;
-        int64_t max_row = 0;
-        for (int64_t row = 1; row < values->length(); ++row) {
-            ColumnarRowRef current(context, row);
-            ColumnarRowRef min_key(context, min_row);
-            ColumnarRowRef max_key(context, max_row);
-            if (key_comparator_->CompareTo(current, min_key) < 0) {
-                min_row = row;
-            }
-            if (key_comparator_->CompareTo(current, max_key) > 0) {
-                max_row = row;
-            }
-        }
-        ColumnarRowRef min_key(context, min_row);
-        ColumnarRowRef max_key(context, max_row);
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InternalRow> copied_min, CopyKey(min_key));
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InternalRow> copied_max, CopyKey(max_key));
-        return std::make_pair(std::move(copied_min), std::move(copied_max));
-    }
+    explicit Impl(std::shared_ptr<arrow::Schema> prepared_schema)
+        : prepared_schema_(std::move(prepared_schema)) {}
 
     Status Write(RealtimeWriteBatch&& write_batch) {
         if (!write_batch.batch || !write_batch.batch->GetData()) {
             return Status::Invalid("PK real-time write batch is null");
         }
         const int64_t row_count = write_batch.batch->GetData()->length;
-        if (row_count <= 0 || write_batch.offset_range.begin < 0 ||
-            write_batch.offset_range.Count() != row_count) {
+        if (write_batch.offset_range.begin < 0 || write_batch.offset_range.Count() != row_count ||
+            row_count <= 0) {
             return Status::Invalid("PK real-time offset range does not match batch row count");
         }
-        const std::vector<RecordBatch::RowKind>& row_kinds = write_batch.batch->GetRowKind();
-        if (!row_kinds.empty() && static_cast<int64_t>(row_kinds.size()) != row_count) {
-            return Status::Invalid("PK real-time row-kind count does not match batch row count");
-        }
-        for (RecordBatch::RowKind row_kind : row_kinds) {
-            PAIMON_ASSIGN_OR_RAISE(const RowKind* validated,
-                                   RowKind::FromByteValue(static_cast<int8_t>(row_kind)));
-            static_cast<void>(validated);
-        }
         PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-            std::shared_ptr<arrow::Array> imported,
+            std::shared_ptr<arrow::Array> array,
             arrow::ImportArray(write_batch.batch->GetData(),
-                               arrow::struct_(write_schema_->fields())));
-        if (!imported || imported->type_id() != arrow::Type::STRUCT) {
-            return Status::Invalid("PK real-time write data is not a StructArray");
+                               arrow::struct_(prepared_schema_->fields())));
+        if (!array || array->type_id() != arrow::Type::STRUCT) {
+            return Status::Invalid("PK real-time prepared batch is not a StructArray");
         }
-        std::shared_ptr<arrow::StructArray> values =
-            checked_pointer_cast<arrow::StructArray>(imported);
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(values->ValidateFull());
-
+        std::shared_ptr<arrow::StructArray> prepared =
+            checked_pointer_cast<arrow::StructArray>(array);
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(prepared->ValidateFull());
         std::lock_guard<std::mutex> lock(mutex_);
         if (last_offset_ && write_batch.offset_range.begin != last_offset_.value()) {
             return Status::Invalid("PK real-time offset ranges must be contiguous");
         }
-        if (row_count > std::numeric_limits<int64_t>::max() - next_sequence_number_) {
-            return Status::Invalid("PK sequence range exceeds INT64_MAX");
-        }
-        auto stored = std::make_shared<const StoredBatch>(
-            StoredBatch{std::move(values), row_kinds, write_batch.offset_range,
-                        next_sequence_number_, GetArrayMemoryUsage(imported->data())});
-        building_batches_.push_back(std::move(stored));
-        building_memory_usage_ += building_batches_.back()->memory_usage;
+        building_.push_back(
+            StoredBatch{prepared, write_batch.offset_range, GetArrayMemoryUsage(prepared->data())});
+        building_memory_usage_ += building_.back().memory_usage;
         last_offset_ = write_batch.offset_range.end;
-        next_sequence_number_ += row_count;
         return Status::OK();
     }
 
     Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>> SealForCommit() {
         std::lock_guard<std::mutex> lock(mutex_);
-        if (building_batches_.empty()) {
+        if (building_.empty()) {
             return std::optional<std::shared_ptr<RealtimeSegmentHandle>>();
         }
-        const OffsetRange range(building_batches_.front()->offset_range.begin,
-                                building_batches_.back()->offset_range.end);
-        auto segment = std::make_shared<Segment>(range, std::move(building_batches_));
-        sealed_segments_.push_back(segment);
-        building_batches_.clear();
+        OffsetRange range(building_.front().offset_range.begin, building_.back().offset_range.end);
+        std::shared_ptr<Segment> segment = std::make_shared<Segment>(range, std::move(building_));
+        sealed_.push_back(segment);
+        building_.clear();
         building_memory_usage_ = 0;
         return std::optional<std::shared_ptr<RealtimeSegmentHandle>>(std::move(segment));
     }
 
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
-        const std::shared_ptr<RealtimeSegmentHandle>& segment) {
-        std::shared_ptr<Segment> typed = std::dynamic_pointer_cast<Segment>(segment);
-        if (!typed) {
+        const std::shared_ptr<RealtimeSegmentHandle>& handle) {
+        std::shared_ptr<Segment> segment = std::dynamic_pointer_cast<Segment>(handle);
+        if (!segment) {
             return Status::Invalid("segment was not created by the PK real-time store");
         }
-        std::vector<std::unique_ptr<BatchReader>> result;
-        result.push_back(std::make_unique<CommitBatchReader>(typed, arrow_pool_));
-        return result;
+        std::vector<std::unique_ptr<BatchReader>> readers;
+        readers.reserve(segment->Batches().size());
+        for (const StoredBatch& batch : segment->Batches()) {
+            readers.push_back(std::make_unique<RawBatchReader>(std::vector<StoredBatch>{batch}));
+        }
+        return readers;
     }
 
     Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() {
         std::lock_guard<std::mutex> lock(mutex_);
-        std::vector<BatchGroup> groups;
-        groups.reserve(sealed_segments_.size() + (building_batches_.empty() ? 0 : 1));
-        for (const std::shared_ptr<Segment>& segment : sealed_segments_) {
-            groups.push_back(segment->Batches());
+        std::vector<std::shared_ptr<Segment>> segments = sealed_;
+        if (!building_.empty()) {
+            OffsetRange range(building_.front().offset_range.begin,
+                              building_.back().offset_range.end);
+            segments.push_back(
+                std::make_shared<Segment>(range, std::vector<StoredBatch>(building_)));
         }
-        if (!building_batches_.empty()) {
-            groups.push_back(building_batches_);
-        }
-        return std::shared_ptr<RealtimeReadView>(new PrimaryKeyRealtimeReadView(std::move(groups)));
+        return std::shared_ptr<RealtimeReadView>(new ReadView(std::move(segments)));
     }
 
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
-        const std::shared_ptr<RealtimeReadView>& view, int64_t lower,
-        const RealtimeQueryContext& context) {
-        std::shared_ptr<PrimaryKeyRealtimeReadView> typed =
-            std::dynamic_pointer_cast<PrimaryKeyRealtimeReadView>(view);
+        const std::shared_ptr<RealtimeReadView>& view, int64_t, const RealtimeQueryContext&) {
+        std::shared_ptr<ReadView> typed = std::dynamic_pointer_cast<ReadView>(view);
         if (!typed) {
             return Status::Invalid("read view was not created by the PK real-time store");
         }
-        if (!context.read_schema || !context.read_schema->release) {
-            return Status::Invalid("PK real-time query read schema is null");
-        }
-        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> requested,
-                                          arrow::ImportSchema(context.read_schema));
-        arrow::FieldVector output_fields = {
-            DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())};
-        arrow::FieldVector aligned_value_fields = write_schema_->fields();
-        std::vector<int32_t> projection = {KeyValueProjectionConsumer::kValueKindProjection};
-        for (const std::shared_ptr<arrow::Field>& field : requested->fields()) {
-            if (field->name() == SpecialFields::ValueKind().Name()) {
-                continue;
-            }
-            output_fields.push_back(field);
-            if (field->name() == SpecialFields::SequenceNumber().Name()) {
-                projection.push_back(KeyValueProjectionConsumer::kSequenceNumberProjection);
-                continue;
-            }
-            int32_t index = FindPkQueryFieldIndex(write_schema_, field);
-            if (index < 0) {
-                Result<int32_t> field_id = NestedProjectionUtils::GetPaimonFieldId(field);
-                if (!field_id.ok()) {
-                    return Status::Invalid(
-                        "PK real-time query field is missing from write schema: ", field->name());
-                }
-                std::string internal_name =
-                    "__paimon_pk_realtime_null_" + std::to_string(field_id.value());
-                while (
-                    NestedProjectionUtils::FindFieldByName(aligned_value_fields, internal_name)) {
-                    internal_name.push_back('_');
-                }
-                index = static_cast<int32_t>(aligned_value_fields.size());
-                aligned_value_fields.push_back(field->WithName(internal_name));
-            } else {
-                aligned_value_fields[index] = write_schema_->field(index)->WithType(field->type());
-            }
-            projection.push_back(index);
+        std::vector<std::unique_ptr<BatchReader>> readers;
+        size_t batch_count = 0;
+        for (const std::shared_ptr<Segment>& segment : typed->Segments()) {
+            batch_count += segment->Batches().size();
         }
-        const std::shared_ptr<arrow::DataType> aligned_value_type =
-            arrow::struct_(aligned_value_fields);
-
-        std::vector<std::unique_ptr<BatchReader>> result;
-        for (const BatchGroup& group : typed->Groups()) {
-            std::vector<std::unique_ptr<KeyValueRecordReader>> batch_readers;
-            std::shared_ptr<InternalRow> min_key;
-            std::shared_ptr<InternalRow> max_key;
-            for (const std::shared_ptr<const StoredBatch>& batch : group) {
-                if (batch->offset_range.end <= lower) {
-                    continue;
-                }
-                const int64_t offset = std::max<int64_t>(0, lower - batch->offset_range.begin);
-                const int64_t length = batch->data->length() - offset;
-                std::shared_ptr<arrow::Array> sliced = batch->data->Slice(offset, length);
-                PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> aligned,
-                                       NestedProjectionUtils::AlignArrayToReadType(
-                                           sliced, aligned_value_type, arrow_pool_.get()));
-                if (!aligned || aligned->type_id() != arrow::Type::STRUCT) {
-                    return Status::Invalid(
-                        "PK real-time query projection did not produce a "
-                        "StructArray");
-                }
-                std::shared_ptr<arrow::StructArray> selected =
-                    checked_pointer_cast<arrow::StructArray>(aligned);
-                using KeyRange =
-                    std::pair<std::shared_ptr<InternalRow>, std::shared_ptr<InternalRow>>;
-                PAIMON_ASSIGN_OR_RAISE(KeyRange key_range, GetKeyRange(selected));
-                if (!min_key || key_comparator_->CompareTo(*key_range.first, *min_key) < 0) {
-                    min_key = key_range.first;
-                }
-                if (!max_key || key_comparator_->CompareTo(*key_range.second, *max_key) > 0) {
-                    max_key = key_range.second;
-                }
-                std::vector<RecordBatch::RowKind> selected_kinds;
-                if (!batch->row_kinds.empty()) {
-                    selected_kinds.assign(batch->row_kinds.begin() + offset,
-                                          batch->row_kinds.end());
-                }
-                std::unique_ptr<KeyValueRecordReader> reader =
-                    std::make_unique<KeyValueInMemoryRecordReader>(
-                        batch->first_sequence_number + offset, selected, selected_kinds,
-                        primary_keys_, /*user_defined_sequence_fields=*/std::vector<std::string>(),
-                        /*sequence_fields_ascending=*/true, key_comparator_, memory_pool_);
-                std::shared_ptr<MergeFunctionWrapper<KeyValue>> batch_merge =
-                    merge_function_wrapper_factory_();
-                if (!batch_merge) {
-                    return Status::Invalid("merge function wrapper factory returned null");
-                }
-                batch_readers.push_back(std::make_unique<MergedKeyValueRecordReader>(
-                    std::move(reader), key_comparator_, batch_merge));
-            }
-            if (batch_readers.empty()) {
-                continue;
-            }
-            std::shared_ptr<MergeFunctionWrapper<KeyValue>> group_merge =
-                merge_function_wrapper_factory_();
-            if (!group_merge) {
-                return Status::Invalid("merge function wrapper factory returned null");
+        readers.reserve(batch_count);
+        for (const std::shared_ptr<Segment>& segment : typed->Segments()) {
+            for (const StoredBatch& batch : segment->Batches()) {
+                readers.push_back(
+                    std::make_unique<RawBatchReader>(std::vector<StoredBatch>{batch}));
             }
-            auto merged = std::make_unique<SortMergeReaderWithLoserTree>(
-                std::move(batch_readers), key_comparator_,
-                /*user_defined_seq_comparator=*/nullptr, group_merge);
-            PAIMON_ASSIGN_OR_RAISE(
-                std::unique_ptr<KeyValueProjectionReader> projected,
-                KeyValueProjectionReader::Create(std::move(merged), arrow::schema(output_fields),
-                                                 projection, read_batch_size_, memory_pool_));
-            result.push_back(
-                std::make_unique<KeyRangeBatchReader>(std::move(projected), min_key, max_key));
         }
-        return result;
+        return readers;
     }
 
-    Status AdvanceCommittedOffset(int64_t committed_end_offset) {
+    Status AdvanceCommittedOffset(int64_t committed_end) {
         std::lock_guard<std::mutex> lock(mutex_);
-        sealed_segments_.erase(
-            std::remove_if(sealed_segments_.begin(), sealed_segments_.end(),
-                           [committed_end_offset](const std::shared_ptr<Segment>& segment) {
-                               return segment->GetOffsetRange().end <= committed_end_offset;
-                           }),
-            sealed_segments_.end());
+        while (!sealed_.empty() && sealed_.front()->GetOffsetRange().end <= committed_end) {
+            sealed_.erase(sealed_.begin());
+        }
         return Status::OK();
     }
 
     uint64_t GetMemoryUsage() const {
         std::lock_guard<std::mutex> lock(mutex_);
-        uint64_t result = building_memory_usage_;
-        for (const std::shared_ptr<Segment>& segment : sealed_segments_) {
-            result += segment->GetMemoryUsage();
+        uint64_t total = building_memory_usage_;
+        for (const std::shared_ptr<Segment>& segment : sealed_) {
+            for (const StoredBatch& batch : segment->Batches()) {
+                total += batch.memory_usage;
+            }
         }
-        return result;
+        return total;
     }
 
  private:
-    std::shared_ptr<arrow::Schema> write_schema_;
-    std::vector<std::string> primary_keys_;
-    std::shared_ptr<FieldsComparator> key_comparator_;
-    std::function<std::shared_ptr<MergeFunctionWrapper<KeyValue>>()>
-        merge_function_wrapper_factory_;
-    int64_t next_sequence_number_;
-    int32_t read_batch_size_;
-    std::shared_ptr<MemoryPool> memory_pool_;
-    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
+    std::shared_ptr<arrow::Schema> prepared_schema_;
     mutable std::mutex mutex_;
-    std::vector<std::shared_ptr<const StoredBatch>> building_batches_;
-    std::vector<std::shared_ptr<Segment>> sealed_segments_;
+    std::vector<StoredBatch> building_;
+    std::vector<std::shared_ptr<Segment>> sealed_;
     uint64_t building_memory_usage_ = 0;
     std::optional<int64_t> last_offset_;
 };
 
-Result<std::shared_ptr<PrimaryKeyRealtimeStore>> PrimaryKeyRealtimeStore::Create(
-    const std::shared_ptr<arrow::Schema>& write_schema,
-    const std::vector<std::string>& primary_keys,
-    const std::shared_ptr<FieldsComparator>& key_comparator,
-    const std::function<std::shared_ptr<MergeFunctionWrapper<KeyValue>>()>&
-        merge_function_wrapper_factory,
-    int64_t restore_max_sequence_number, int32_t read_batch_size,
-    const std::shared_ptr<MemoryPool>& memory_pool) {
-    if (!write_schema || primary_keys.empty() || !key_comparator ||
-        !merge_function_wrapper_factory || !memory_pool || read_batch_size <= 0) {
-        return Status::Invalid("PK real-time store requires schema, keys, merge helpers, and pool");
-    }
-    if (restore_max_sequence_number < -1) {
-        return Status::Invalid("PK restore max sequence number must be at least -1");
-    }
-    if (restore_max_sequence_number == std::numeric_limits<int64_t>::max()) {
-        return Status::Invalid("PK sequence number has reached INT64_MAX");
-    }
-    for (const std::string& key : primary_keys) {
-        if (write_schema->GetFieldIndex(key) < 0) {
-            return Status::Invalid("primary key ", key, " is missing from write schema");
-        }
-    }
-    auto impl = std::make_unique<Impl>(
-        write_schema, primary_keys, key_comparator, merge_function_wrapper_factory,
-        restore_max_sequence_number + 1, read_batch_size, memory_pool);
-    return std::shared_ptr<PrimaryKeyRealtimeStore>(new PrimaryKeyRealtimeStore(std::move(impl)));
-}
-
 PrimaryKeyRealtimeStore::PrimaryKeyRealtimeStore(std::unique_ptr<Impl>&& impl)
     : impl_(std::move(impl)) {}
-
 PrimaryKeyRealtimeStore::~PrimaryKeyRealtimeStore() = default;
 
+Result<std::shared_ptr<PrimaryKeyRealtimeStore>> PrimaryKeyRealtimeStore::Create(
+    const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::shared_ptr<MemoryPool>& memory_pool) {
+    if (!prepared_schema || !memory_pool) {
+        return Status::Invalid("PK prepared schema or memory pool is null");
+    }
+    return std::shared_ptr<PrimaryKeyRealtimeStore>(
+        new PrimaryKeyRealtimeStore(std::make_unique<Impl>(prepared_schema)));
+}
 Status PrimaryKeyRealtimeStore::Write(RealtimeWriteBatch&& batch) {
     return impl_->Write(std::move(batch));
 }
-
 Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>>
 PrimaryKeyRealtimeStore::SealForCommit() {
     return impl_->SealForCommit();
 }
-
 Result<std::vector<std::unique_ptr<BatchReader>>> PrimaryKeyRealtimeStore::CreateCommitReaders(
     const std::shared_ptr<RealtimeSegmentHandle>& segment) {
     return impl_->CreateCommitReaders(segment);
 }
-
 Result<std::shared_ptr<RealtimeReadView>> PrimaryKeyRealtimeStore::AcquireReadView() {
     return impl_->AcquireReadView();
 }
-
 Result<std::vector<std::unique_ptr<BatchReader>>> PrimaryKeyRealtimeStore::CreateQueryReaders(
-    const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
+    const std::shared_ptr<RealtimeReadView>& view, int64_t offset,
     const RealtimeQueryContext& context) {
-    return impl_->CreateQueryReaders(view, offset_begin, context);
+    return impl_->CreateQueryReaders(view, offset, context);
 }
-
-Status PrimaryKeyRealtimeStore::AdvanceCommittedOffset(int64_t committed_offset) {
-    return impl_->AdvanceCommittedOffset(committed_offset);
+Status PrimaryKeyRealtimeStore::AdvanceCommittedOffset(int64_t offset) {
+    return impl_->AdvanceCommittedOffset(offset);
 }
-
 uint64_t PrimaryKeyRealtimeStore::GetMemoryUsage() const {
     return impl_->GetMemoryUsage();
 }
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.h b/src/paimon/core/realtime/primary_key_realtime_store.h
index 017864c04..5e18dd74f 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.h
+++ b/src/paimon/core/realtime/primary_key_realtime_store.h
@@ -19,11 +19,7 @@
 
 #pragma once
 
-#include <cstdint>
-#include <functional>
 #include <memory>
-#include <string>
-#include <vector>
 
 #include "paimon/realtime/realtime_store.h"
 
@@ -34,34 +30,15 @@ class Schema;
 namespace paimon {
 
 class CoreOptions;
-class FieldsComparator;
-struct KeyValue;
 class MemoryPool;
-class InternalRow;
-template <typename T>
-class MergeFunctionWrapper;
 
 Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options);
 
-/// Optional metadata exposed by PK query readers with a known inclusive key range.
-class PrimaryKeyRangeProvider {
- public:
-    virtual ~PrimaryKeyRangeProvider() = default;
-
-    virtual std::shared_ptr<InternalRow> GetMinKey() const = 0;
-    virtual std::shared_ptr<InternalRow> GetMaxKey() const = 0;
-};
-
-/// In-memory store for primary-key real-time writes.
+/// In-memory store for prepared primary-key real-time batches.
 class PrimaryKeyRealtimeStore final : public RealtimeStore {
  public:
     static Result<std::shared_ptr<PrimaryKeyRealtimeStore>> Create(
-        const std::shared_ptr<arrow::Schema>& write_schema,
-        const std::vector<std::string>& primary_keys,
-        const std::shared_ptr<FieldsComparator>& key_comparator,
-        const std::function<std::shared_ptr<MergeFunctionWrapper<KeyValue>>()>&
-            merge_function_wrapper_factory,
-        int64_t restore_max_sequence_number, int32_t read_batch_size,
+        const std::shared_ptr<arrow::Schema>& prepared_schema,
         const std::shared_ptr<MemoryPool>& memory_pool);
 
     ~PrimaryKeyRealtimeStore() override;
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index 66901a6b1..43831d7be 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -9,23 +9,18 @@
  *
  *   http://www.apache.org/licenses/LICENSE-2.0
  *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 #include "paimon/core/realtime/primary_key_realtime_store.h"
 
-#include <cstdint>
-#include <limits>
-#include <map>
 #include <memory>
 #include <optional>
 #include <string>
-#include <utility>
 #include <vector>
 
 #include "arrow/api.h"
@@ -33,14 +28,52 @@
 #include "arrow/ipc/json_simple.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
-#include "paimon/common/utils/fields_comparator.h"
+#include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/core/core_options.h"
-#include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
-#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
+#include "paimon/core/realtime/realtime_fields.h"
+#include "paimon/macros.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/testing/utils/testharness.h"
 
 namespace paimon::test {
+namespace {
+
+std::shared_ptr<arrow::Schema> PreparedSchema() {
+    return arrow::schema(
+        {DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
+         DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
+             ->WithNullable(false),
+         DataField::ConvertDataFieldToArrowField(RealtimeOffsetField())->WithNullable(false),
+         arrow::field("id", arrow::int64()), arrow::field("value", arrow::utf8())});
+}
+
+std::unique_ptr<RecordBatch> MakeBatch(const std::string& json) {
+    std::shared_ptr<arrow::Array> array =
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(PreparedSchema()->fields()), json)
+            .ValueOrDie();
+    auto c_array = std::make_unique<ArrowArray>();
+    EXPECT_TRUE(arrow::ExportArray(*array, c_array.get()).ok());
+    return RecordBatchBuilder(c_array.get()).Finish().value();
+}
+
+Result<std::string> ReadJson(const std::vector<std::unique_ptr<BatchReader>>& readers) {
+    std::vector<std::shared_ptr<arrow::Array>> batches;
+    for (const std::unique_ptr<BatchReader>& reader : readers) {
+        while (true) {
+            PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader->NextBatch());
+            if (BatchReader::IsEofBatch(batch)) {
+                break;
+            }
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+                std::shared_ptr<arrow::Array> array,
+                arrow::ImportArray(batch.first.get(), batch.second.get()));
+            batches.push_back(std::move(array));
+        }
+    }
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> result,
+                                      arrow::Concatenate(batches));
+    return result->ToString();
+}
 
 TEST(PrimaryKeyRealtimeStoreOptionsTest, TestSupportedOptions) {
     ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
@@ -66,428 +99,69 @@ TEST(PrimaryKeyRealtimeStoreOptionsTest, TestUnsupportedOptions) {
     }
 }
 
-class PrimaryKeyRealtimeStoreTest : public testing::Test {
- public:
-    void SetUp() override {
-        pool_ = std::shared_ptr<MemoryPool>(GetMemoryPool());
-        schema_ = arrow::schema(
-            {arrow::field("id", arrow::int64()), arrow::field("value", arrow::utf8())});
-        ASSERT_OK_AND_ASSIGN(store_, CreateStore(schema_, {"id"}, /*restore_max_sequence=*/4));
-    }
-
-    Result<std::shared_ptr<PrimaryKeyRealtimeStore>> CreateStore(
-        const std::shared_ptr<arrow::Schema>& schema, const std::vector<std::string>& primary_keys,
-        int64_t restore_max_sequence) const {
-        std::vector<DataField> key_fields;
-        key_fields.reserve(primary_keys.size());
-        for (const std::string& primary_key : primary_keys) {
-            const int32_t index = schema->GetFieldIndex(primary_key);
-            key_fields.emplace_back(index, schema->field(index));
-        }
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FieldsComparator> key_comparator,
-                               FieldsComparator::Create(key_fields,
-                                                        /*is_ascending_order=*/true));
-        auto merge_factory = []() {
-            auto merge_function =
-                std::make_unique<DeduplicateMergeFunction>(/*ignore_delete=*/false);
-            return std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge_function));
-        };
-        return PrimaryKeyRealtimeStore::Create(schema, primary_keys, key_comparator, merge_factory,
-                                               restore_max_sequence,
-                                               /*read_batch_size=*/2, pool_);
-    }
-
-    std::unique_ptr<RecordBatch> MakeBatch(
-        const std::string& json, const std::vector<RecordBatch::RowKind>& row_kinds = {},
-        const std::shared_ptr<arrow::Schema>& schema = nullptr) const {
-        const std::shared_ptr<arrow::Schema>& batch_schema = schema ? schema : schema_;
-        std::shared_ptr<arrow::Array> array =
-            arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(batch_schema->fields()), json)
-                .ValueOrDie();
-        ArrowArray c_array;
-        EXPECT_TRUE(arrow::ExportArray(*array, &c_array).ok());
-        RecordBatchBuilder builder(&c_array);
-        builder.SetRowKinds(row_kinds);
-        return builder.Finish().value();
-    }
-
-    std::unique_ptr<ArrowSchema> MakeReadSchema(const arrow::FieldVector& fields) const {
-        auto c_schema = std::make_unique<ArrowSchema>();
-        EXPECT_TRUE(arrow::ExportSchema(*arrow::schema(fields), c_schema.get()).ok());
-        return c_schema;
-    }
-
-    void AssertReaderOutput(const std::vector<std::unique_ptr<BatchReader>>& readers,
-                            const std::shared_ptr<arrow::DataType>& type,
-                            const std::string& json) const {
-        std::vector<std::shared_ptr<arrow::Array>> batches;
-        for (const std::unique_ptr<BatchReader>& reader : readers) {
-            while (true) {
-                ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch());
-                if (BatchReader::IsEofBatch(batch)) {
-                    break;
-                }
-                arrow::Result<std::shared_ptr<arrow::Array>> imported =
-                    arrow::ImportArray(batch.first.get(), batch.second.get());
-                ASSERT_TRUE(imported.ok()) << imported.status().ToString();
-                batches.push_back(std::move(imported).ValueOrDie());
-            }
-        }
-        ASSERT_FALSE(batches.empty());
-        arrow::Result<std::shared_ptr<arrow::Array>> concatenated = arrow::Concatenate(batches);
-        ASSERT_TRUE(concatenated.ok()) << concatenated.status().ToString();
-        std::shared_ptr<arrow::Array> actual = std::move(concatenated).ValueOrDie();
-        std::shared_ptr<arrow::Array> expected =
-            arrow::ipc::internal::json::ArrayFromJSON(type, json).ValueOrDie();
-        ASSERT_TRUE(actual->Equals(*expected))
-            << "expected: " << expected->ToString() << ", actual: " << actual->ToString();
-        for (const std::unique_ptr<BatchReader>& reader : readers) {
-            reader->Close();
-        }
-    }
-
-    std::shared_ptr<arrow::DataType> CommitType() const {
-        return arrow::struct_({
-            DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind()),
-            schema_->field(0),
-            schema_->field(1),
-        });
-    }
-
-    std::shared_ptr<arrow::DataType> QueryType() const {
-        return arrow::struct_({
-            DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind()),
-            DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber()),
-            schema_->field(0),
-            schema_->field(1),
-        });
-    }
-
-    arrow::FieldVector FullQueryFields(
-        const std::shared_ptr<arrow::Schema>& schema = nullptr) const {
-        const std::shared_ptr<arrow::Schema>& query_schema = schema ? schema : schema_;
-        arrow::FieldVector fields = {
-            DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())};
-        fields.insert(fields.end(), query_schema->fields().begin(), query_schema->fields().end());
-        return fields;
-    }
-
- protected:
-    std::shared_ptr<MemoryPool> pool_;
-    std::shared_ptr<arrow::Schema> schema_;
-    std::shared_ptr<PrimaryKeyRealtimeStore> store_;
-};
-
-TEST_F(PrimaryKeyRealtimeStoreTest, TestWriteAndSeal) {
+TEST(PrimaryKeyRealtimeStoreTest, TestWriteAndSealValidation) {
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                         PrimaryKeyRealtimeStore::Create(PreparedSchema(), GetDefaultPool()));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
-                         store_->SealForCommit());
+                         store->SealForCommit());
     ASSERT_FALSE(segment.has_value());
-    ASSERT_NOK_WITH_MSG(store_->Write(RealtimeWriteBatch{nullptr, OffsetRange(0, 0)}),
+    ASSERT_NOK_WITH_MSG(store->Write(RealtimeWriteBatch{nullptr, OffsetRange(0, 0)}),
                         "write batch is null");
     ASSERT_NOK_WITH_MSG(
-        store_->Write(RealtimeWriteBatch{MakeBatch(R"([[1, "a"], [2, "b"]])"), OffsetRange(0, 0)}),
+        store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 1, 0, 1, "one"]])"), OffsetRange(0, 0)}),
         "offset range does not match batch row count");
 
-    ASSERT_OK(
-        store_->Write(RealtimeWriteBatch{MakeBatch(R"([[1, "a"], [2, "b"]])"), OffsetRange(0, 2)}));
-    ASSERT_NOK_WITH_MSG(
-        store_->Write(RealtimeWriteBatch{MakeBatch(R"([[4, "d"]])"), OffsetRange(3, 4)}),
-        "offset ranges must be contiguous");
-    ASSERT_OK(store_->Write(RealtimeWriteBatch{MakeBatch(R"([[3, "c"]])"), OffsetRange(2, 3)}));
+    ASSERT_OK(store->Write(RealtimeWriteBatch{
+        MakeBatch(R"([[0, 1, 0, 1, "one"], [0, 2, 1, 2, "two"]])"), OffsetRange(0, 2)}));
+    ASSERT_NOK_WITH_MSG(store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 3, 3, 3, "three"]])"),
+                                                        OffsetRange(3, 4)}),
+                        "offset ranges must be contiguous");
+    ASSERT_OK(store->Write(
+        RealtimeWriteBatch{MakeBatch(R"([[0, 3, 2, 3, "three"]])"), OffsetRange(2, 3)}));
 
-    ASSERT_OK_AND_ASSIGN(segment, store_->SealForCommit());
+    ASSERT_OK_AND_ASSIGN(segment, store->SealForCommit());
     ASSERT_TRUE(segment.has_value());
     ASSERT_EQ(OffsetRange(0, 3), segment.value()->GetOffsetRange());
-    ASSERT_GT(store_->GetMemoryUsage(), 0);
-
-    struct ValidationCase {
-        int64_t restore_max_sequence;
-        std::string error;
-    };
-    const std::vector<ValidationCase> cases = {
-        {-2, "restore max sequence number must be at least -1"},
-        {std::numeric_limits<int64_t>::max(), "sequence number has reached INT64_MAX"},
-    };
-    for (const ValidationCase& test_case : cases) {
-        ASSERT_NOK_WITH_MSG(CreateStore(schema_, {"id"}, test_case.restore_max_sequence),
-                            test_case.error);
-    }
-}
-
-TEST_F(PrimaryKeyRealtimeStoreTest, TestCommitBatches) {
-    ASSERT_OK(store_->Write(RealtimeWriteBatch{
-        MakeBatch(R"([[3, "three"], [1, "before"]])",
-                  {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::UPDATE_BEFORE}),
-        OffsetRange(0, 2)}));
-    ASSERT_OK(store_->Write(RealtimeWriteBatch{
-        MakeBatch(R"([[2, "after"]])", {RecordBatch::RowKind::UPDATE_AFTER}), OffsetRange(2, 3)}));
-    ASSERT_OK(store_->Write(
-        RealtimeWriteBatch{MakeBatch(R"([[4, "deleted"], [0, "zero"]])",
-                                     {RecordBatch::RowKind::DELETE, RecordBatch::RowKind::INSERT}),
-                           OffsetRange(3, 5)}));
-    ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
-                         store_->SealForCommit());
-    ASSERT_TRUE(segment.has_value());
-    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
-                         store_->CreateCommitReaders(segment.value()));
-    AssertReaderOutput(readers, CommitType(),
-                       R"([[0, 3, "three"], [1, 1, "before"], [2, 2, "after"],
-                            [3, 4, "deleted"], [0, 0, "zero"]])");
-
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store_->AcquireReadView());
-    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(FullQueryFields());
-    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
-                                 /*enable_predicate_pushdown=*/false};
-    ASSERT_OK_AND_ASSIGN(readers, store_->CreateQueryReaders(view, /*offset_begin=*/0, context));
-    AssertReaderOutput(readers, QueryType(),
-                       R"([[0, 9, 0, "zero"], [1, 6, 1, "before"], [2, 7, 2, "after"],
-                            [0, 5, 3, "three"], [3, 8, 4, "deleted"]])");
-}
-
-TEST_F(PrimaryKeyRealtimeStoreTest, TestMutationMerge) {
-    ASSERT_OK(store_->Write(
-        RealtimeWriteBatch{MakeBatch(R"([[1, "old"], [2, "two"]])"), OffsetRange(0, 2)}));
-    ASSERT_OK(store_->Write(RealtimeWriteBatch{
-        MakeBatch(R"([[1, "new"], [2, "gone"]])",
-                  {RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::DELETE}),
-        OffsetRange(2, 4)}));
-
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store_->AcquireReadView());
-    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(FullQueryFields());
-    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
-                                 /*enable_predicate_pushdown=*/false};
-    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
-                         store_->CreateQueryReaders(view, /*offset_begin=*/0, context));
-    AssertReaderOutput(readers, QueryType(), R"([[2, 7, 1, "new"], [3, 8, 2, "gone"]])");
+    ASSERT_GT(store->GetMemoryUsage(), 0);
+    ASSERT_NOK_WITH_MSG(
+        store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 4, 4, 4, "four"]])"), OffsetRange(4, 5)}),
+        "offset ranges must be contiguous");
+    ASSERT_OK(store->Write(
+        RealtimeWriteBatch{MakeBatch(R"([[0, 4, 3, 4, "four"]])"), OffsetRange(3, 4)}));
 }
 
-TEST_F(PrimaryKeyRealtimeStoreTest, TestReadViewLifecycle) {
-    ASSERT_OK(store_->Write(RealtimeWriteBatch{MakeBatch(R"([[10, "a"], [11, "b"], [12, "c"]])"),
-                                               OffsetRange(10, 13)}));
+TEST(PrimaryKeyRealtimeStoreTest, TestCommitBatches) {
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                         PrimaryKeyRealtimeStore::Create(PreparedSchema(), GetDefaultPool()));
+    ASSERT_OK(store->Write(RealtimeWriteBatch{
+        MakeBatch(R"([[0, 5, 0, 3, "three"], [1, 6, 1, 1, "before"]])"), OffsetRange(0, 2)}));
+    ASSERT_OK(store->Write(
+        RealtimeWriteBatch{MakeBatch(R"([[2, 7, 2, 2, "after"]])"), OffsetRange(2, 3)}));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
-                         store_->SealForCommit());
+                         store->SealForCommit());
     ASSERT_TRUE(segment.has_value());
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store_->AcquireReadView());
-    ASSERT_EQ(std::optional<OffsetRange>(OffsetRange(10, 13)), view->GetOffsetRange());
-
-    ASSERT_OK(store_->AdvanceCommittedOffset(13));
-    ASSERT_EQ(0, store_->GetMemoryUsage());
-    ASSERT_OK(
-        store_->Write(RealtimeWriteBatch{MakeBatch(R"([[13, "later"]])"), OffsetRange(13, 14)}));
-
-    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(FullQueryFields());
-    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
-                                 /*enable_predicate_pushdown=*/false};
-    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
-                         store_->CreateQueryReaders(view, /*offset_begin=*/11, context));
-    AssertReaderOutput(readers, QueryType(), R"([[0, 6, 11, "b"], [0, 7, 12, "c"]])");
-
-    std::unique_ptr<ArrowSchema> empty_schema = MakeReadSchema(FullQueryFields());
-    context.read_schema = empty_schema.get();
-    ASSERT_OK_AND_ASSIGN(readers, store_->CreateQueryReaders(view, /*offset_begin=*/13, context));
-    ASSERT_TRUE(readers.empty());
-}
-
-TEST_F(PrimaryKeyRealtimeStoreTest, TestQueryKeyRange) {
-    ASSERT_OK(store_->Write(
-        RealtimeWriteBatch{MakeBatch(R"([[5, "five"], [1, "one"]])"), OffsetRange(0, 2)}));
-    ASSERT_OK(store_->SealForCommit());
-    ASSERT_OK(store_->Write(
-        RealtimeWriteBatch{MakeBatch(R"([[9, "nine"], [7, "seven"]])"), OffsetRange(2, 4)}));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store_->AcquireReadView());
-    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(FullQueryFields());
-    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
-                                 /*enable_predicate_pushdown=*/false};
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
-                         store_->CreateQueryReaders(view, /*offset_begin=*/0, context));
-    ASSERT_EQ(2, readers.size());
-    const std::vector<std::pair<int64_t, int64_t>> key_ranges = {{1, 5}, {7, 9}};
-    for (size_t i = 0; i < readers.size(); ++i) {
-        auto* range = dynamic_cast<PrimaryKeyRangeProvider*>(readers[i].get());
-        ASSERT_NE(nullptr, range);
-        ASSERT_EQ(key_ranges[i].first, range->GetMinKey()->GetLong(0));
-        ASSERT_EQ(key_ranges[i].second, range->GetMaxKey()->GetLong(0));
-    }
-    AssertReaderOutput(readers, QueryType(),
-                       R"([[0, 6, 1, "one"], [0, 5, 5, "five"], [0, 8, 7, "seven"],
-                            [0, 7, 9, "nine"]])");
-
-    ASSERT_OK(store_->AdvanceCommittedOffset(2));
-    ASSERT_OK_AND_ASSIGN(view, store_->AcquireReadView());
-    read_schema = MakeReadSchema(FullQueryFields());
-    context.read_schema = read_schema.get();
-    ASSERT_OK_AND_ASSIGN(readers, store_->CreateQueryReaders(view, /*offset_begin=*/0, context));
-    ASSERT_EQ(1, readers.size());
-    auto* range = dynamic_cast<PrimaryKeyRangeProvider*>(readers[0].get());
-    ASSERT_NE(nullptr, range);
-    ASSERT_EQ(7, range->GetMinKey()->GetLong(0));
-    ASSERT_EQ(9, range->GetMaxKey()->GetLong(0));
-    AssertReaderOutput(readers, QueryType(), R"([[0, 8, 7, "seven"], [0, 7, 9, "nine"]])");
+                         store->CreateCommitReaders(segment.value()));
+    ASSERT_OK_AND_ASSIGN(std::string actual, ReadJson(readers));
+    ASSERT_EQ(
+        "-- is_valid: all not null\n-- child 0 type: int8\n  [\n    0,\n    1,\n    2\n  ]\n-- "
+        "child 1 type: int64\n  [\n    5,\n    6,\n    7\n  ]\n-- child 2 type: int64\n  [\n    "
+        "0,\n    1,\n    2\n  ]\n-- child 3 type: int64\n  [\n    3,\n    1,\n    2\n  ]\n-- child "
+        "4 type: string\n  [\n    \"three\",\n    \"before\",\n    \"after\"\n  ]",
+        actual);
 }
 
-TEST_F(PrimaryKeyRealtimeStoreTest, TestSequenceExhaustion) {
-    const int64_t max_sequence = std::numeric_limits<int64_t>::max();
+TEST(PrimaryKeyRealtimeStoreTest, TestReclaimKeepsReadView) {
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         CreateStore(schema_, {"id"}, max_sequence - 3));
-    ASSERT_OK(store->Write(RealtimeWriteBatch{MakeBatch(R"([[1, "kept"]])"), OffsetRange(10, 11)}));
-    ASSERT_NOK_WITH_MSG(
-        store->Write(RealtimeWriteBatch{
-            MakeBatch(R"([[7, "rejected-a"], [8, "rejected-b"], [9, "rejected-c"]])"),
-            OffsetRange(11, 14)}),
-        "sequence range exceeds INT64_MAX");
+                         PrimaryKeyRealtimeStore::Create(PreparedSchema(), GetDefaultPool()));
     ASSERT_OK(
-        store->Write(RealtimeWriteBatch{MakeBatch(R"([[2, "also-kept"]])"), OffsetRange(11, 12)}));
-
+        store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 0, 4, 1, "one"]])"), OffsetRange(4, 5)}));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
                          store->SealForCommit());
-    ASSERT_TRUE(segment.has_value());
-    ASSERT_EQ(OffsetRange(10, 12), segment.value()->GetOffsetRange());
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
-    ASSERT_EQ(std::optional<OffsetRange>(OffsetRange(10, 12)), view->GetOffsetRange());
-    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(FullQueryFields());
-    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
-                                 /*enable_predicate_pushdown=*/false};
-    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
-                         store->CreateQueryReaders(view, /*offset_begin=*/10, context));
-    AssertReaderOutput(readers, QueryType(),
-                       R"([[0, 9223372036854775805, 1, "kept"],
-                            [0, 9223372036854775806, 2, "also-kept"]])");
-}
-
-TEST_F(PrimaryKeyRealtimeStoreTest, TestQueryProjection) {
-    ASSERT_OK(
-        store_->Write(RealtimeWriteBatch{MakeBatch(R"([[2, "b"], [1, "a"]])"), OffsetRange(0, 2)}));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store_->AcquireReadView());
-    const std::shared_ptr<arrow::Field> value_kind =
-        DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind());
-    const std::shared_ptr<arrow::Field> sequence =
-        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber());
-    struct ProjectionCase {
-        arrow::FieldVector requested;
-        std::shared_ptr<arrow::DataType> expected_type;
-        std::string expected_json;
-    };
-    const std::vector<ProjectionCase> cases = {
-        {{schema_->field(1), value_kind, sequence, schema_->field(0)},
-         arrow::struct_({value_kind, schema_->field(1), sequence, schema_->field(0)}),
-         R"([[0, "a", 6, 1], [0, "b", 5, 2]])"},
-        {{schema_->field(0), value_kind},
-         arrow::struct_({value_kind, schema_->field(0)}),
-         R"([[0, 1], [0, 2]])"},
-    };
-    for (const ProjectionCase& test_case : cases) {
-        std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(test_case.requested);
-        RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
-                                     /*enable_predicate_pushdown=*/false};
-        ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
-                             store_->CreateQueryReaders(view, /*offset_begin=*/0, context));
-        AssertReaderOutput(readers, test_case.expected_type, test_case.expected_json);
-    }
-
-    std::unique_ptr<ArrowSchema> read_schema =
-        MakeReadSchema({arrow::field("unknown", arrow::int64())});
-    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
-                                 /*enable_predicate_pushdown=*/false};
-    ASSERT_NOK_WITH_MSG(store_->CreateQueryReaders(view, /*offset_begin=*/0, context),
-                        "query field is missing from write schema: unknown");
-}
-
-TEST_F(PrimaryKeyRealtimeStoreTest, TestQueryProjectionMatchesRenamedFieldsById) {
-    const std::shared_ptr<arrow::Field> id =
-        DataField::ConvertDataFieldToArrowField(DataField(0, arrow::field("id", arrow::int64())));
-    const std::shared_ptr<arrow::Field> value =
-        DataField::ConvertDataFieldToArrowField(DataField(1, arrow::field("value", arrow::utf8())));
-    const std::shared_ptr<arrow::Schema> write_schema = arrow::schema({id, value});
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         CreateStore(write_schema, {"id"}, /*restore_max_sequence=*/4));
-    ASSERT_OK(store->Write(
-        RealtimeWriteBatch{MakeBatch(R"([[1, "kept"]])", {}, write_schema), OffsetRange(0, 1)}));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
-
-    const std::shared_ptr<arrow::Field> renamed_value = DataField::ConvertDataFieldToArrowField(
-        DataField(1, arrow::field("renamed", arrow::utf8())));
-    const std::shared_ptr<arrow::Field> renamed_id = DataField::ConvertDataFieldToArrowField(
-        DataField(0, arrow::field("renamed_id", arrow::int64())));
-    const std::shared_ptr<arrow::Field> replaced =
-        DataField::ConvertDataFieldToArrowField(DataField(2, arrow::field("value", arrow::utf8())));
-    const std::shared_ptr<arrow::Field> replaced_id =
-        DataField::ConvertDataFieldToArrowField(DataField(4, arrow::field("id", arrow::int64())));
-    const std::shared_ptr<arrow::Field> added =
-        DataField::ConvertDataFieldToArrowField(DataField(3, arrow::field("added", arrow::utf8())));
-    std::unique_ptr<ArrowSchema> read_schema =
-        MakeReadSchema({renamed_value, renamed_id, replaced, replaced_id, added});
-    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
-                                 /*enable_predicate_pushdown=*/false};
-    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
-                         store->CreateQueryReaders(view, /*offset_begin=*/0, context));
-    const std::shared_ptr<arrow::DataType> result_type =
-        arrow::struct_({DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind()),
-                        renamed_value, renamed_id, replaced, replaced_id, added});
-    AssertReaderOutput(readers, result_type, R"([[0, "kept", 1, null, null, null]])");
-}
-
-TEST_F(PrimaryKeyRealtimeStoreTest, TestNestedProjection) {
-    const std::shared_ptr<arrow::Field> id =
-        DataField::ConvertDataFieldToArrowField(DataField(0, arrow::field("id", arrow::int64())));
-    const std::shared_ptr<arrow::Field> a =
-        DataField::ConvertDataFieldToArrowField(DataField(10, arrow::field("a", arrow::int64())));
-    const std::shared_ptr<arrow::Field> b =
-        DataField::ConvertDataFieldToArrowField(DataField(11, arrow::field("b", arrow::int64())));
-    const std::shared_ptr<arrow::Field> payload = DataField::ConvertDataFieldToArrowField(
-        DataField(1, arrow::field("payload", arrow::struct_({a, b}))));
-    const std::shared_ptr<arrow::Schema> nested_schema = arrow::schema({id, payload});
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         CreateStore(nested_schema, {"id"}, /*restore_max_sequence=*/4));
-    ASSERT_OK(store->Write(RealtimeWriteBatch{
-        MakeBatch(R"([[2, [200, 2000]], [1, [100, null]], [3, [300, 3000]]])", {}, nested_schema),
-        OffsetRange(0, 3)}));
-
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
-    const std::shared_ptr<arrow::Field> projected_payload = payload->WithType(arrow::struct_({b}));
-    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema({projected_payload});
-    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
-                                 /*enable_predicate_pushdown=*/false};
-    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
-                         store->CreateQueryReaders(view, /*offset_begin=*/0, context));
-    const std::shared_ptr<arrow::DataType> result_type = arrow::struct_(
-        {DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind()), projected_payload});
-    AssertReaderOutput(readers, result_type, R"([[0, [null]], [0, [2000]], [0, [3000]]])");
-}
-
-TEST_F(PrimaryKeyRealtimeStoreTest, TestCompositeKeyClipping) {
-    std::shared_ptr<arrow::Schema> composite_schema =
-        arrow::schema({arrow::field("id", arrow::int64()), arrow::field("region", arrow::utf8()),
-                       arrow::field("value", arrow::utf8())});
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         CreateStore(composite_schema, {"id", "region"},
-                                     /*restore_max_sequence=*/4));
-    ASSERT_OK(store->Write(
-        RealtimeWriteBatch{MakeBatch(R"([[9, "z", "clipped"], [2, "b", "two-b"], [1, "c", "one-c"],
-                       [2, "a", "two-a"]])",
-                                     {}, composite_schema),
-                           OffsetRange(20, 24)}));
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
-    const std::shared_ptr<arrow::Field> sequence =
-        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber());
-    std::unique_ptr<ArrowSchema> read_schema =
-        MakeReadSchema({sequence, composite_schema->field(0), composite_schema->field(2)});
-    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
-                                 /*enable_predicate_pushdown=*/false};
-    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
-                         store->CreateQueryReaders(view, /*offset_begin=*/21, context));
-    ASSERT_EQ(1, readers.size());
-    auto* range = dynamic_cast<PrimaryKeyRangeProvider*>(readers[0].get());
-    ASSERT_NE(nullptr, range);
-    ASSERT_EQ(1, range->GetMinKey()->GetLong(0));
-    ASSERT_EQ("c", range->GetMinKey()->GetString(1).ToString());
-    ASSERT_EQ(2, range->GetMaxKey()->GetLong(0));
-    ASSERT_EQ("b", range->GetMaxKey()->GetString(1).ToString());
-    std::shared_ptr<arrow::DataType> query_type =
-        arrow::struct_({DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind()),
-                        sequence, composite_schema->field(0), composite_schema->field(2)});
-    AssertReaderOutput(readers, query_type,
-                       R"([[0, 7, 1, "one-c"], [0, 8, 2, "two-a"],
-                            [0, 6, 2, "two-b"]])");
+    ASSERT_OK(store->AdvanceCommittedOffset(5));
+    ASSERT_EQ(std::optional<OffsetRange>(OffsetRange(4, 5)), view->GetOffsetRange());
 }
 
+}  // namespace
 }  // namespace paimon::test
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index 066e54e8a..b73cfdb8a 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -42,7 +42,6 @@
 #include "paimon/status.h"
 
 namespace paimon {
-
 RealtimeContextImpl::RealtimeContextImpl(const std::shared_ptr<RealtimeStoreFactory>& factory)
     : factory_(factory) {}
 
@@ -83,26 +82,6 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     std::lock_guard<std::mutex> registry_lock(mutex_);
     const RealtimePartitionBucket key(request.partition, request.bucket);
     auto iter = stores_.find(key);
-    std::optional<int64_t> initial_max_sequence_number;
-    PrimaryKeyRealtimeStoreCreateConfig* primary_key_config =
-        std::get_if<PrimaryKeyRealtimeStoreCreateConfig>(&request.mode_config);
-    if (primary_key_config) {
-        auto [sequence_iter, inserted] = materialized_max_sequence_numbers_.emplace(
-            key, primary_key_config->restore_max_sequence_number);
-        if (!inserted && primary_key_config->restore_max_sequence_number > sequence_iter->second) {
-            if (iter != stores_.end()) {
-                if (request.write_schema) {
-                    ArrowSchemaRelease(request.write_schema.get());
-                }
-                return Status::Invalid(
-                    "restore max sequence number exceeds the materialized watermark of an "
-                    "existing PK real-time store");
-            }
-            sequence_iter->second = primary_key_config->restore_max_sequence_number;
-        }
-        initial_max_sequence_number = sequence_iter->second;
-        primary_key_config->restore_max_sequence_number = sequence_iter->second;
-    }
     int64_t initial_offset = 0;
     auto offset_iter = committed_offsets_.find(key);
     if (offset_iter != committed_offsets_.end()) {
@@ -134,7 +113,13 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
                 initial_offset = memory_range->end;
             }
         }
-        return RealtimeStoreState{iter->second, initial_offset, initial_max_sequence_number};
+        return RealtimeStoreState{iter->second, initial_offset};
+    }
+    if (!request.memory_pool) {
+        if (request.write_schema) {
+            ArrowSchemaRelease(request.write_schema.get());
+        }
+        return Status::Invalid("real-time store memory pool is null");
     }
     Result<std::shared_ptr<RealtimeStore>> store_result = factory_->Create(std::move(request));
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> store, std::move(store_result));
@@ -142,17 +127,7 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     if (offset_iter != committed_offsets_.end()) {
         reclaimed_offsets_.emplace(key, offset_iter->second);
     }
-    return RealtimeStoreState{std::move(store), initial_offset, initial_max_sequence_number};
-}
-
-void RealtimeContextImpl::AdvanceMaterializedMaxSequenceNumber(
-    const RealtimePartitionBucket& partition_bucket, int64_t max_sequence_number) {
-    std::lock_guard<std::mutex> lock(mutex_);
-    auto [iter, inserted] =
-        materialized_max_sequence_numbers_.emplace(partition_bucket, max_sequence_number);
-    if (!inserted && max_sequence_number > iter->second) {
-        iter->second = max_sequence_number;
-    }
+    return RealtimeStoreState{std::move(store), initial_offset};
 }
 
 Result<std::vector<RealtimePartitionBucketView>> RealtimeContextImpl::AcquireReadViews() {
diff --git a/src/paimon/core/realtime/realtime_context_impl.h b/src/paimon/core/realtime/realtime_context_impl.h
index f4cd3866e..4f62cf1ee 100644
--- a/src/paimon/core/realtime/realtime_context_impl.h
+++ b/src/paimon/core/realtime/realtime_context_impl.h
@@ -47,7 +47,6 @@ class MemoryPool;
 struct RealtimeStoreState {
     std::shared_ptr<RealtimeStore> store;
     int64_t initial_offset;
-    std::optional<int64_t> initial_max_sequence_number;
 };
 
 struct RealtimePartitionBucketView {
@@ -68,9 +67,6 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
 
     Result<RealtimeStoreState> GetOrCreateRealtimeStore(RealtimeStoreCreateRequest&& request);
 
-    void AdvanceMaterializedMaxSequenceNumber(const RealtimePartitionBucket& partition_bucket,
-                                              int64_t max_sequence_number);
-
     Result<std::vector<RealtimePartitionBucketView>> AcquireReadViews();
 
     Result<std::string> PinReadView(const RealtimePartitionBucketView& view, int64_t ttl_millis);
@@ -100,7 +96,6 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
     std::mutex mutex_;
     std::mutex progress_mutex_;
     std::map<RealtimePartitionBucket, std::shared_ptr<RealtimeStore>> stores_;
-    std::map<RealtimePartitionBucket, int64_t> materialized_max_sequence_numbers_;
     // Full-table progress used as the initial offset when a store is created lazily.
     RealtimeOffsetMap committed_offsets_;
     // Progress already reflected in stores owned by this context.
diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp
index ab0abe4a7..07bbf555b 100644
--- a/src/paimon/core/realtime/realtime_context_test.cpp
+++ b/src/paimon/core/realtime/realtime_context_test.cpp
@@ -9,12 +9,11 @@
  *
  *   http://www.apache.org/licenses/LICENSE-2.0
  *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 #include <chrono>
@@ -31,7 +30,6 @@
 #include "arrow/c/helpers.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
 #include "paimon/memory/memory_pool.h"
-#include "paimon/realtime/realtime_store.h"
 #include "paimon/testing/utils/testharness.h"
 
 namespace paimon::test {
@@ -49,26 +47,21 @@ class TestingRealtimeStore : public RealtimeStore {
     Status Write(RealtimeWriteBatch&&) override {
         return Status::OK();
     }
-
     Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>> SealForCommit() override {
         return std::optional<std::shared_ptr<RealtimeSegmentHandle>>();
     }
-
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
         const std::shared_ptr<RealtimeSegmentHandle>&) override {
         return std::vector<std::unique_ptr<BatchReader>>();
     }
-
     Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
         ++acquire_count;
         return std::make_shared<TestingReadView>();
     }
-
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
         const std::shared_ptr<RealtimeReadView>&, int64_t, const RealtimeQueryContext&) override {
         return std::vector<std::unique_ptr<BatchReader>>();
     }
-
     Status AdvanceCommittedOffset(int64_t committed_offset) override {
         ++advance_count;
         if (fail_next_advance) {
@@ -78,7 +71,6 @@ class TestingRealtimeStore : public RealtimeStore {
         committed_offsets.push_back(committed_offset);
         return Status::OK();
     }
-
     uint64_t GetMemoryUsage() const override {
         return 0;
     }
@@ -105,11 +97,11 @@ class TestingRealtimeStoreFactory : public RealtimeStoreFactory {
 };
 
 std::unique_ptr<ArrowSchema> MakeWriteSchema() {
-    auto c_schema = std::make_unique<ArrowSchema>();
+    auto schema = std::make_unique<ArrowSchema>();
     EXPECT_TRUE(
-        arrow::ExportSchema(*arrow::schema({arrow::field("id", arrow::int64())}), c_schema.get())
+        arrow::ExportSchema(*arrow::schema({arrow::field("id", arrow::int64())}), schema.get())
             .ok());
-    return c_schema;
+    return schema;
 }
 
 Result<std::shared_ptr<RealtimeContextImpl>> CreateContext(
@@ -129,41 +121,29 @@ Result<RealtimeStoreState> GetOrCreateAppendStore(
                                    AppendRealtimeStoreCreateConfig{StatisticsMode::NONE}});
 }
 
-Result<RealtimeStoreState> GetOrCreatePrimaryKeyStore(
-    const std::shared_ptr<RealtimeContextImpl>& context,
-    const std::map<std::string, std::string>& partition, int32_t bucket,
-    int64_t restore_max_sequence_number, const std::shared_ptr<MemoryPool>& memory_pool) {
-    return context->GetOrCreateRealtimeStore(RealtimeStoreCreateRequest{
-        MakeWriteSchema(), /*options=*/{}, memory_pool, partition, bucket,
-        PrimaryKeyRealtimeStoreCreateConfig{{"id"}, restore_max_sequence_number}});
-}
-
-TEST(RealtimeContextTest, TestReusesIndexerAndCapturesRegisteredViews) {
+TEST(RealtimeContextTest, TestReusesStoreAndCapturesRegisteredViews) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
-    std::shared_ptr<MemoryPool> pool = GetDefaultPool();
-
-    ASSERT_OK_AND_ASSIGN(RealtimeStoreState first_state,
+    ASSERT_OK_AND_ASSIGN(RealtimeStoreState first,
                          GetOrCreateAppendStore(context, {{"dt", "2026-08-02"}}, 0,
-                                                MakeWriteSchema(), {{"k", "v"}}, pool));
-    ASSERT_EQ(0, first_state.initial_offset);
-    ASSERT_FALSE(first_state.initial_max_sequence_number);
-    ASSERT_OK_AND_ASSIGN(
-        RealtimeStoreState first_again_state,
-        GetOrCreateAppendStore(context, {{"dt", "2026-08-02"}}, 0, MakeWriteSchema(), {}, pool));
-    ASSERT_EQ(first_state.store, first_again_state.store);
-    ASSERT_EQ(0, first_again_state.initial_offset);
+                                                MakeWriteSchema(), {{"k", "v"}}, GetDefaultPool()));
+    ASSERT_EQ(0, first.initial_offset);
+    ASSERT_OK_AND_ASSIGN(RealtimeStoreState second,
+                         GetOrCreateAppendStore(context, {{"dt", "2026-08-02"}}, 0,
+                                                MakeWriteSchema(), {}, GetDefaultPool()));
+    ASSERT_EQ(first.store, second.store);
+    ASSERT_EQ(0, second.initial_offset);
     ASSERT_EQ(1, factory->stores.size());
     ASSERT_EQ(1, factory->stores[0]->acquire_count);
 
-    ASSERT_OK_AND_ASSIGN(
-        RealtimeStoreState second_state,
-        GetOrCreateAppendStore(context, {{"dt", "2026-08-02"}}, 1, MakeWriteSchema(), {}, pool));
-    ASSERT_OK_AND_ASSIGN(
-        RealtimeStoreState third_state,
-        GetOrCreateAppendStore(context, {{"dt", "2026-08-03"}}, 0, MakeWriteSchema(), {}, pool));
-    ASSERT_NE(first_state.store, second_state.store);
-    ASSERT_NE(first_state.store, third_state.store);
+    ASSERT_OK_AND_ASSIGN(RealtimeStoreState third,
+                         GetOrCreateAppendStore(context, {{"dt", "2026-08-02"}}, 1,
+                                                MakeWriteSchema(), {}, GetDefaultPool()));
+    ASSERT_OK_AND_ASSIGN(RealtimeStoreState fourth,
+                         GetOrCreateAppendStore(context, {{"dt", "2026-08-03"}}, 0,
+                                                MakeWriteSchema(), {}, GetDefaultPool()));
+    ASSERT_NE(first.store, third.store);
+    ASSERT_NE(first.store, fourth.store);
     ASSERT_EQ(3, factory->stores.size());
 
     ASSERT_OK_AND_ASSIGN(std::vector<RealtimePartitionBucketView> views,
@@ -171,57 +151,22 @@ TEST(RealtimeContextTest, TestReusesIndexerAndCapturesRegisteredViews) {
     ASSERT_EQ(3, views.size());
     const RealtimePartitionBucket expected_partition_bucket({{"dt", "2026-08-02"}}, 0);
     ASSERT_EQ(expected_partition_bucket, views[0].partition_bucket);
-    ASSERT_EQ(first_state.store, views[0].store);
+    ASSERT_EQ(first.store, views[0].store);
     ASSERT_TRUE(views[0].read_view);
     ASSERT_EQ(2, factory->stores[0]->acquire_count);
     ASSERT_EQ(1, factory->stores[1]->acquire_count);
     ASSERT_EQ(1, factory->stores[2]->acquire_count);
 }
 
-TEST(RealtimeContextTest, TestReconcilesPrimaryKeyInitialSequence) {
-    auto factory = std::make_shared<TestingRealtimeStoreFactory>();
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
-    const std::map<std::string, std::string> partition = {{"dt", "2026-08-02"}};
-
-    ASSERT_OK_AND_ASSIGN(
-        RealtimeStoreState first_state,
-        GetOrCreatePrimaryKeyStore(context, partition, /*bucket=*/0,
-                                   /*restore_max_sequence_number=*/4, GetDefaultPool()));
-    ASSERT_EQ(4, first_state.initial_max_sequence_number);
-
-    const RealtimePartitionBucket partition_bucket(partition, /*bucket=*/0);
-    context->AdvanceMaterializedMaxSequenceNumber(partition_bucket, /*max_sequence_number=*/8);
-    ASSERT_OK_AND_ASSIGN(
-        RealtimeStoreState retained_state,
-        GetOrCreatePrimaryKeyStore(context, partition, /*bucket=*/0,
-                                   /*restore_max_sequence_number=*/6, GetDefaultPool()));
-    ASSERT_EQ(first_state.store, retained_state.store);
-    ASSERT_EQ(8, retained_state.initial_max_sequence_number);
-
-    ASSERT_NOK_WITH_MSG(
-        GetOrCreatePrimaryKeyStore(context, partition, /*bucket=*/0,
-                                   /*restore_max_sequence_number=*/10, GetDefaultPool()),
-        "restore max sequence number exceeds the materialized watermark of an "
-        "existing PK real-time store");
-
-    const RealtimePartitionBucket new_partition_bucket(partition, /*bucket=*/1);
-    context->AdvanceMaterializedMaxSequenceNumber(new_partition_bucket,
-                                                  /*max_sequence_number=*/8);
-    ASSERT_OK_AND_ASSIGN(
-        RealtimeStoreState new_state,
-        GetOrCreatePrimaryKeyStore(context, partition, /*bucket=*/1,
-                                   /*restore_max_sequence_number=*/10, GetDefaultPool()));
-    ASSERT_EQ(10, new_state.initial_max_sequence_number);
-}
-
 TEST(RealtimeContextTest, TestCommittedProgressIsMonotonicAndSelective) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
-    std::shared_ptr<MemoryPool> pool = GetDefaultPool();
     const std::map<std::string, std::string> partition = {{"dt", "2026-08-02"}};
 
-    ASSERT_OK(GetOrCreateAppendStore(context, partition, 0, MakeWriteSchema(), {}, pool));
-    ASSERT_OK(GetOrCreateAppendStore(context, partition, 1, MakeWriteSchema(), {}, pool));
+    ASSERT_OK(
+        GetOrCreateAppendStore(context, partition, 0, MakeWriteSchema(), {}, GetDefaultPool()));
+    ASSERT_OK(
+        GetOrCreateAppendStore(context, partition, 1, MakeWriteSchema(), {}, GetDefaultPool()));
     ASSERT_EQ(2, factory->stores.size());
 
     ASSERT_NOK_WITH_MSG(context->AdvanceCommittedProgress(-1, {}),
@@ -238,9 +183,9 @@ TEST(RealtimeContextTest, TestCommittedProgressIsMonotonicAndSelective) {
     ASSERT_EQ(std::vector<int64_t>({7}), factory->stores[0]->committed_offsets);
     ASSERT_TRUE(factory->stores[1]->committed_offsets.empty());
 
-    ASSERT_OK_AND_ASSIGN(
-        RealtimeStoreState restored_state,
-        GetOrCreateAppendStore(context, {{"dt", "unknown"}}, 0, MakeWriteSchema(), {}, pool));
+    ASSERT_OK_AND_ASSIGN(RealtimeStoreState restored_state,
+                         GetOrCreateAppendStore(context, {{"dt", "unknown"}}, 0, MakeWriteSchema(),
+                                                {}, GetDefaultPool()));
     ASSERT_EQ(9, restored_state.initial_offset);
 
     ASSERT_OK(context->AdvanceCommittedProgress(
@@ -260,12 +205,14 @@ TEST(RealtimeContextTest, TestCommittedProgressIsMonotonicAndSelective) {
 TEST(RealtimeContextTest, TestRetriesOnlyIncompleteReclamation) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
-    std::shared_ptr<MemoryPool> pool = GetDefaultPool();
     const std::map<std::string, std::string> partition = {{"dt", "2026-08-02"}};
 
-    ASSERT_OK(GetOrCreateAppendStore(context, partition, 0, MakeWriteSchema(), {}, pool));
-    ASSERT_OK(GetOrCreateAppendStore(context, partition, 1, MakeWriteSchema(), {}, pool));
-    ASSERT_OK(GetOrCreateAppendStore(context, partition, 2, MakeWriteSchema(), {}, pool));
+    ASSERT_OK(
+        GetOrCreateAppendStore(context, partition, 0, MakeWriteSchema(), {}, GetDefaultPool()));
+    ASSERT_OK(
+        GetOrCreateAppendStore(context, partition, 1, MakeWriteSchema(), {}, GetDefaultPool()));
+    ASSERT_OK(
+        GetOrCreateAppendStore(context, partition, 2, MakeWriteSchema(), {}, GetDefaultPool()));
     ASSERT_EQ(3, factory->stores.size());
     factory->stores[1]->fail_next_advance = true;
 
@@ -281,7 +228,7 @@ TEST(RealtimeContextTest, TestRetriesOnlyIncompleteReclamation) {
 
     ASSERT_OK_AND_ASSIGN(
         RealtimeStoreState failed_store_state,
-        GetOrCreateAppendStore(context, partition, 1, MakeWriteSchema(), {}, pool));
+        GetOrCreateAppendStore(context, partition, 1, MakeWriteSchema(), {}, GetDefaultPool()));
     ASSERT_EQ(8, failed_store_state.initial_offset);
 
     ASSERT_OK(context->AdvanceCommittedProgress(5, committed_offsets));
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index 65bcebcad..c85ff6322 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -9,12 +9,11 @@
  *
  *   http://www.apache.org/licenses/LICENSE-2.0
  *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 #include "paimon/core/realtime/realtime_primary_key_writer.h"
@@ -26,95 +25,254 @@
 
 #include "arrow/api.h"
 #include "arrow/c/bridge.h"
-#include "paimon/common/reader/concat_batch_reader.h"
+#include "arrow/compute/api.h"
 #include "paimon/common/table/special_fields.h"
+#include "paimon/common/types/data_field.h"
 #include "paimon/common/types/row_kind.h"
-#include "paimon/common/utils/arrow/arrow_utils.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/scope_guard.h"
-#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/io/merged_key_value_record_reader.h"
+#include "paimon/core/io/prepared_key_value_reader.h"
+#include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
+#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
 #include "paimon/core/mergetree/merge_tree_writer.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
+#include "paimon/core/realtime/realtime_fields.h"
 #include "paimon/core/utils/commit_increment.h"
 #include "paimon/macros.h"
-#include "paimon/realtime/realtime_context.h"
 
 namespace paimon {
 
+namespace {
+
+struct PreparedArrayPrivateData {
+    void (*release)(ArrowArray*);
+    void* private_data;
+    std::shared_ptr<arrow::MemoryPool> arrow_pool;
+};
+
+void ReleasePreparedArray(ArrowArray* array) {
+    auto* data = static_cast<PreparedArrayPrivateData*>(array->private_data);
+    array->release = data->release;
+    array->private_data = data->private_data;
+    array->release(array);
+    delete data;
+}
+
+Status RetainPreparedArrayPool(ArrowArray* array,
+                               const std::shared_ptr<arrow::MemoryPool>& arrow_pool) {
+    if (!array || !array->release || !arrow_pool) {
+        return Status::Invalid("cannot retain prepared batch memory pool");
+    }
+    array->private_data =
+        new PreparedArrayPrivateData{array->release, array->private_data, arrow_pool};
+    array->release = ReleasePreparedArray;
+    return Status::OK();
+}
+
+Result<std::shared_ptr<arrow::StructArray>> PrepareBatch(
+    std::unique_ptr<RecordBatch>&& batch, const std::shared_ptr<arrow::Schema>& write_schema,
+    const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::vector<std::string>& trimmed_primary_keys, int64_t first_sequence_number,
+    int64_t first_offset, arrow::MemoryPool* arrow_pool) {
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+        std::shared_ptr<arrow::Array> input,
+        arrow::ImportArray(batch->GetData(), arrow::struct_(write_schema->fields())));
+    if (!input || input->type_id() != arrow::Type::STRUCT) {
+        return Status::Invalid("PK real-time write data is not a StructArray");
+    }
+    std::shared_ptr<arrow::StructArray> values = checked_pointer_cast<arrow::StructArray>(input);
+    const int64_t count = values->length();
+    arrow::Int8Builder kinds(arrow_pool);
+    arrow::Int64Builder sequences(arrow_pool);
+    arrow::Int64Builder offsets(arrow_pool);
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(kinds.Reserve(count));
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(sequences.Reserve(count));
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets.Reserve(count));
+    const std::vector<RecordBatch::RowKind>& row_kinds = batch->GetRowKind();
+    for (int64_t row = 0; row < count; ++row) {
+        const RecordBatch::RowKind kind =
+            row_kinds.empty() ? RecordBatch::RowKind::INSERT : row_kinds[row];
+        kinds.UnsafeAppend(static_cast<int8_t>(kind));
+        sequences.UnsafeAppend(first_sequence_number + row);
+        offsets.UnsafeAppend(first_offset + row);
+    }
+    std::shared_ptr<arrow::Array> kind_array;
+    std::shared_ptr<arrow::Array> sequence_array;
+    std::shared_ptr<arrow::Array> offset_array;
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(kinds.Finish(&kind_array));
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(sequences.Finish(&sequence_array));
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets.Finish(&offset_array));
+    arrow::ArrayVector columns = {std::move(kind_array), std::move(sequence_array),
+                                  std::move(offset_array)};
+    columns.insert(columns.end(), values->fields().begin(), values->fields().end());
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+        std::shared_ptr<arrow::StructArray> prepared,
+        arrow::StructArray::Make(std::move(columns), prepared_schema->fields()));
+
+    std::vector<arrow::compute::SortKey> sort_keys;
+    sort_keys.reserve(trimmed_primary_keys.size() + 1);
+    for (const std::string& key : trimmed_primary_keys) {
+        sort_keys.emplace_back(key, arrow::compute::SortOrder::Ascending);
+    }
+    sort_keys.emplace_back(SpecialFields::SequenceNumber().Name(),
+                           arrow::compute::SortOrder::Ascending);
+    arrow::compute::ExecContext context(arrow_pool);
+    arrow::compute::SortOptions options(sort_keys, arrow::compute::NullPlacement::AtStart);
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+        arrow::Datum indices,
+        arrow::compute::SortIndices(arrow::Datum(prepared), options, &context));
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+        arrow::Datum sorted,
+        arrow::compute::Take(arrow::Datum(prepared), indices,
+                             arrow::compute::TakeOptions::NoBoundsCheck(), &context));
+    std::shared_ptr<arrow::Array> sorted_array = sorted.make_array();
+    if (!sorted_array || sorted_array->type_id() != arrow::Type::STRUCT) {
+        return Status::Invalid("PK real-time sorted batch is not a StructArray");
+    }
+    return checked_pointer_cast<arrow::StructArray>(std::move(sorted_array));
+}
+
+}  // namespace
+
 Result<std::shared_ptr<RealtimePrimaryKeyWriter>> RealtimePrimaryKeyWriter::Create(
-    const std::map<std::string, std::string>& partition, int32_t bucket,
     const std::shared_ptr<arrow::Schema>& write_schema,
-    const std::shared_ptr<RealtimeContextImpl>& realtime_context,
-    const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
-    const std::shared_ptr<MemoryPool>& memory_pool, const RealtimeStoreState& store_state) {
-    return std::shared_ptr<RealtimePrimaryKeyWriter>(
-        new RealtimePrimaryKeyWriter(store_state.store, merge_tree_writer, realtime_context,
-                                     RealtimePartitionBucket(partition, bucket), write_schema,
-                                     store_state.initial_offset, memory_pool));
+    const std::vector<std::string>& trimmed_primary_keys,
+    const std::shared_ptr<FieldsComparator>& key_comparator, const RealtimeStoreState& store_state,
+    int64_t restored_max_sequence_number, const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
+    const std::shared_ptr<MemoryPool>& memory_pool) {
+    if (!store_state.store || !merge_tree_writer || !write_schema || !key_comparator ||
+        !memory_pool) {
+        return Status::Invalid("PK real-time writer received a null dependency");
+    }
+    if (trimmed_primary_keys.empty()) {
+        return Status::Invalid("PK real-time writer requires at least one primary key");
+    }
+    if (restored_max_sequence_number < -1 ||
+        restored_max_sequence_number == std::numeric_limits<int64_t>::max()) {
+        return Status::Invalid("PK restored sequence number is invalid");
+    }
+    arrow::FieldVector key_fields;
+    key_fields.reserve(trimmed_primary_keys.size());
+    for (const std::string& key : trimmed_primary_keys) {
+        std::shared_ptr<arrow::Field> field = write_schema->GetFieldByName(key);
+        if (!field) {
+            return Status::Invalid("PK field is missing from write schema: ", key);
+        }
+        key_fields.push_back(std::move(field));
+    }
+    arrow::FieldVector prepared_fields = {
+        DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
+        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
+            ->WithNullable(false),
+        DataField::ConvertDataFieldToArrowField(RealtimeOffsetField())->WithNullable(false)};
+    prepared_fields.insert(prepared_fields.end(), write_schema->fields().begin(),
+                           write_schema->fields().end());
+    return std::shared_ptr<RealtimePrimaryKeyWriter>(new RealtimePrimaryKeyWriter(
+        store_state.store, merge_tree_writer, write_schema,
+        arrow::schema(std::move(prepared_fields)), arrow::schema(std::move(key_fields)),
+        trimmed_primary_keys, key_comparator, store_state.initial_offset,
+        restored_max_sequence_number, memory_pool));
 }
 
 RealtimePrimaryKeyWriter::RealtimePrimaryKeyWriter(
     const std::shared_ptr<RealtimeStore>& realtime_store,
     const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
-    const std::shared_ptr<RealtimeContextImpl>& realtime_context,
-    const RealtimePartitionBucket& partition_bucket,
-    const std::shared_ptr<arrow::Schema>& write_schema, int64_t next_offset,
-    const std::shared_ptr<MemoryPool>& memory_pool)
+    const std::shared_ptr<arrow::Schema>& write_schema,
+    const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::shared_ptr<arrow::Schema>& key_schema,
+    const std::vector<std::string>& trimmed_primary_keys,
+    const std::shared_ptr<FieldsComparator>& key_comparator, int64_t next_offset,
+    int64_t last_sequence_number, const std::shared_ptr<MemoryPool>& memory_pool)
     : memory_pool_(memory_pool),
+      arrow_pool_(GetArrowPool(memory_pool)),
       realtime_store_(realtime_store),
       merge_tree_writer_(merge_tree_writer),
-      realtime_context_(realtime_context),
-      partition_bucket_(partition_bucket),
       write_schema_(write_schema),
-      next_offset_(next_offset) {}
+      prepared_schema_(prepared_schema),
+      key_schema_(key_schema),
+      trimmed_primary_keys_(trimmed_primary_keys),
+      key_comparator_(key_comparator),
+      next_offset_(next_offset),
+      last_sequence_number_(last_sequence_number) {}
 
 Status RealtimePrimaryKeyWriter::Write(std::unique_ptr<RecordBatch>&& batch) {
     if (!batch || !batch->GetData()) {
         return Status::Invalid("PK real-time write batch is null");
     }
-    const int64_t row_count = batch->GetData()->length;
-    if (row_count == 0) {
+    const int64_t count = batch->GetData()->length;
+    if (count == 0) {
         return Status::OK();
     }
+    const std::vector<RecordBatch::RowKind>& row_kinds = batch->GetRowKind();
+    if (!row_kinds.empty() && static_cast<int64_t>(row_kinds.size()) != count) {
+        return Status::Invalid("PK real-time row-kind count does not match batch row count");
+    }
+    for (RecordBatch::RowKind row_kind : row_kinds) {
+        PAIMON_ASSIGN_OR_RAISE(const RowKind* validated,
+                               RowKind::FromByteValue(static_cast<int8_t>(row_kind)));
+        static_cast<void>(validated);
+    }
     std::lock_guard<std::mutex> lock(realtime_store_mutex_);
-    if (row_count > std::numeric_limits<int64_t>::max() - next_offset_) {
+    if (count > std::numeric_limits<int64_t>::max() - next_offset_) {
         return Status::Invalid("real-time offset range exceeds INT64_MAX");
     }
-    const OffsetRange range(next_offset_, next_offset_ + row_count);
-    PAIMON_RETURN_NOT_OK(realtime_store_->Write(RealtimeWriteBatch{std::move(batch), range}));
-    next_offset_ += row_count;
+    // Reserve INT64_MAX as the exhausted sequence-number sentinel.
+    if (last_sequence_number_ >= std::numeric_limits<int64_t>::max() - count) {
+        return Status::Invalid("PK sequence range exceeds INT64_MAX");
+    }
+    const int64_t first_sequence = last_sequence_number_ + 1;
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<arrow::StructArray> prepared,
+        PrepareBatch(std::move(batch), write_schema_, prepared_schema_, trimmed_primary_keys_,
+                     first_sequence, next_offset_, arrow_pool_.get()));
+    auto output = std::make_unique<ArrowArray>();
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*prepared, output.get()));
+    PAIMON_RETURN_NOT_OK(RetainPreparedArrayPool(output.get(), arrow_pool_));
+    RecordBatchBuilder builder(output.get());
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RecordBatch> prepared_batch, builder.Finish());
+    PAIMON_RETURN_NOT_OK(realtime_store_->Write(RealtimeWriteBatch{
+        std::move(prepared_batch), OffsetRange(next_offset_, next_offset_ + count)}));
+    next_offset_ += count;
+    last_sequence_number_ += count;
     return Status::OK();
 }
 
 Result<CommitIncrement> RealtimePrimaryKeyWriter::PrepareCommit(bool wait_compaction) {
-    std::lock_guard<std::mutex> lock(prepare_mutex_);
+    std::lock_guard<std::mutex> prepare_lock(prepare_mutex_);
     std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment;
     {
-        std::lock_guard<std::mutex> realtime_store_lock(realtime_store_mutex_);
-        PAIMON_ASSIGN_OR_RAISE(std::optional<std::shared_ptr<RealtimeSegmentHandle>> sealed_segment,
+        std::lock_guard<std::mutex> store_lock(realtime_store_mutex_);
+        PAIMON_ASSIGN_OR_RAISE(std::optional<std::shared_ptr<RealtimeSegmentHandle>> sealed,
                                realtime_store_->SealForCommit());
-        segment = std::move(sealed_segment);
+        segment = std::move(sealed);
+    }
+    if (segment && !segment.value()) {
+        return Status::Invalid("PK real-time store sealed a null segment");
     }
+    std::optional<OffsetRange> sealed_range;
+    int64_t expected_raw_row_count = 0;
     if (segment) {
-        PAIMON_RETURN_NOT_OK(FlushSegment(segment.value()));
+        sealed_range = segment.value()->GetOffsetRange();
+        if (sealed_range->begin < 0 || sealed_range->end < sealed_range->begin ||
+            __builtin_sub_overflow(sealed_range->end, sealed_range->begin,
+                                   &expected_raw_row_count)) {
+            return Status::Invalid("PK real-time store returned an invalid sealed offset range");
+        }
+        PAIMON_RETURN_NOT_OK(FlushSegment(segment.value(), expected_raw_row_count));
     }
     PAIMON_ASSIGN_OR_RAISE(CommitIncrement increment,
                            merge_tree_writer_->PrepareCommit(wait_compaction));
     if (segment) {
-        const std::vector<std::shared_ptr<DataFileMeta>>& new_files =
-            increment.GetNewFilesIncrement().NewFiles();
-        if (!new_files.empty()) {
-            realtime_context_->AdvanceMaterializedMaxSequenceNumber(
-                partition_bucket_, DataFileMeta::GetMaxSequenceNumber(new_files));
-        }
-        increment.SetRealtimeOffsetRange(segment.value()->GetOffsetRange());
+        increment.SetRealtimeOffsetRange(sealed_range.value());
     }
     return increment;
 }
 
-Status RealtimePrimaryKeyWriter::FlushSegment(
-    const std::shared_ptr<RealtimeSegmentHandle>& segment) {
+Status RealtimePrimaryKeyWriter::FlushSegment(const std::shared_ptr<RealtimeSegmentHandle>& segment,
+                                              int64_t expected_raw_row_count) {
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
                            realtime_store_->CreateCommitReaders(segment));
     ScopeGuard readers_guard([&readers]() {
@@ -124,72 +282,26 @@ Status RealtimePrimaryKeyWriter::FlushSegment(
             }
         }
     });
-    for (const std::unique_ptr<BatchReader>& reader : readers) {
+    int64_t raw_row_count = 0;
+    std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
+    sorted_readers.reserve(readers.size());
+    for (std::unique_ptr<BatchReader>& reader : readers) {
         if (!reader) {
             return Status::Invalid("PK real-time store returned a null commit reader");
         }
+        PAIMON_ASSIGN_OR_RAISE(
+            std::unique_ptr<KeyValueRecordReader> prepared_reader,
+            AdaptPreparedBatchReader(std::move(reader), prepared_schema_, std::nullopt, key_schema_,
+                                     write_schema_, memory_pool_, &raw_row_count));
+        auto merge_function = std::make_unique<DeduplicateMergeFunction>(/*ignore_delete=*/false);
+        sorted_readers.push_back(std::make_unique<MergedKeyValueRecordReader>(
+            std::move(prepared_reader), key_comparator_,
+            std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge_function))));
     }
-    ConcatBatchReader reader(std::move(readers), memory_pool_);
-    ScopeGuard reader_guard([&reader]() { reader.Close(); });
-    const OffsetRange offset_range = segment->GetOffsetRange();
-    int64_t emitted_rows = 0;
-    while (true) {
-        PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader.NextBatch());
-        if (BatchReader::IsEofBatch(batch)) {
-            break;
-        }
-        auto& [c_array, c_schema] = batch;
-        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> imported,
-                                          arrow::ImportArray(c_array.get(), c_schema.get()));
-        if (!imported || imported->type_id() != arrow::Type::STRUCT) {
-            return Status::Invalid("PK real-time store commit reader returned a non-StructArray");
-        }
-        std::shared_ptr<arrow::StructArray> struct_array =
-            checked_pointer_cast<arrow::StructArray>(imported);
-        std::shared_ptr<arrow::Array> value_kind =
-            struct_array->GetFieldByName(SpecialFields::ValueKind().Name());
-        if (!value_kind || value_kind->type_id() != arrow::Type::INT8) {
-            return Status::Invalid(
-                "PK real-time store commit reader must return an INT8 _VALUE_KIND field");
-        }
-        std::shared_ptr<arrow::Int8Array> encoded_row_kinds =
-            checked_pointer_cast<arrow::Int8Array>(value_kind);
-        std::vector<RecordBatch::RowKind> row_kinds;
-        row_kinds.reserve(static_cast<size_t>(encoded_row_kinds->length()));
-        for (int64_t i = 0; i < encoded_row_kinds->length(); ++i) {
-            if (encoded_row_kinds->IsNull(i)) {
-                return Status::Invalid("PK real-time store commit reader returned a null row kind");
-            }
-            PAIMON_ASSIGN_OR_RAISE(const RowKind* row_kind,
-                                   RowKind::FromByteValue(encoded_row_kinds->Value(i)));
-            row_kinds.push_back(static_cast<RecordBatch::RowKind>(row_kind->ToByteValue()));
-        }
-        PAIMON_ASSIGN_OR_RAISE(struct_array, ArrowUtils::RemoveFieldFromStructArray(
-                                                 struct_array, SpecialFields::ValueKind().Name()));
-        if (!struct_array->type()->Equals(arrow::struct_(write_schema_->fields()))) {
-            return Status::Invalid(
-                "PK real-time store commit reader schema does not match table write schema");
-        }
-        const int64_t row_count = struct_array->length();
-        if (row_count > offset_range.Count() - emitted_rows) {
-            return Status::Invalid(
-                "PK real-time store commit readers returned more rows than the sealed offset "
-                "range");
-        }
-        emitted_rows += row_count;
-        if (row_count == 0) {
-            continue;
-        }
-        auto output = std::make_unique<ArrowArray>();
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*struct_array, output.get()));
-        RecordBatchBuilder builder(output.get());
-        builder.SetRowKinds(row_kinds);
-        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RecordBatch> record_batch, builder.Finish());
-        PAIMON_RETURN_NOT_OK(merge_tree_writer_->Write(std::move(record_batch)));
-    }
-    if (emitted_rows != offset_range.Count()) {
-        return Status::Invalid(
-            "PK real-time store commit readers returned fewer rows than the sealed offset range");
+    readers_guard.Release();
+    PAIMON_RETURN_NOT_OK(merge_tree_writer_->WriteSortedReaders(std::move(sorted_readers)));
+    if (raw_row_count != expected_raw_row_count) {
+        return Status::Invalid("PK real-time store commit readers did not cover the sealed range");
     }
     return Status::OK();
 }
@@ -197,27 +309,21 @@ Status RealtimePrimaryKeyWriter::FlushSegment(
 Status RealtimePrimaryKeyWriter::Compact(bool) {
     return Status::Invalid("PK real-time write does not support explicit compaction");
 }
-
 uint64_t RealtimePrimaryKeyWriter::GetMemoryUsage() const {
     return realtime_store_->GetMemoryUsage();
 }
-
 Status RealtimePrimaryKeyWriter::FlushMemory() {
     return Status::OK();
 }
-
 Result<bool> RealtimePrimaryKeyWriter::CompactNotCompleted() {
     return merge_tree_writer_->CompactNotCompleted();
 }
-
 Status RealtimePrimaryKeyWriter::Sync() {
     return merge_tree_writer_->Sync();
 }
-
 Status RealtimePrimaryKeyWriter::Close() {
     return merge_tree_writer_->Close();
 }
-
 std::shared_ptr<Metrics> RealtimePrimaryKeyWriter::GetMetrics() const {
     return merge_tree_writer_->GetMetrics();
 }
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.h b/src/paimon/core/realtime/realtime_primary_key_writer.h
index c1e893c85..6abb1ccd0 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.h
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.h
@@ -20,16 +20,16 @@
 #pragma once
 
 #include <cstdint>
-#include <map>
 #include <memory>
 #include <mutex>
 #include <string>
+#include <vector>
 
 #include "paimon/core/utils/batch_writer.h"
-#include "paimon/realtime/realtime_context.h"
 #include "paimon/realtime/realtime_store.h"
 
 namespace arrow {
+class MemoryPool;
 class Schema;
 }  // namespace arrow
 
@@ -37,18 +37,19 @@ namespace paimon {
 
 class MemoryPool;
 class MergeTreeWriter;
-class RealtimeContextImpl;
+class FieldsComparator;
 struct RealtimeStoreState;
 
-/// Primary-key real-time writer backed by an in-memory mutation indexer.
+/// Coordinates framework-prepared primary-key real-time writes.
 class RealtimePrimaryKeyWriter final : public BatchWriter {
  public:
     static Result<std::shared_ptr<RealtimePrimaryKeyWriter>> Create(
-        const std::map<std::string, std::string>& partition, int32_t bucket,
         const std::shared_ptr<arrow::Schema>& write_schema,
-        const std::shared_ptr<RealtimeContextImpl>& realtime_context,
+        const std::vector<std::string>& trimmed_primary_keys,
+        const std::shared_ptr<FieldsComparator>& key_comparator,
+        const RealtimeStoreState& store_state, int64_t restore_max_sequence_number,
         const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
-        const std::shared_ptr<MemoryPool>& memory_pool, const RealtimeStoreState& store_state);
+        const std::shared_ptr<MemoryPool>& memory_pool);
 
     Status Write(std::unique_ptr<RecordBatch>&& batch) override;
     Result<CommitIncrement> PrepareCommit(bool wait_compaction) override;
@@ -63,20 +64,28 @@ class RealtimePrimaryKeyWriter final : public BatchWriter {
  private:
     RealtimePrimaryKeyWriter(const std::shared_ptr<RealtimeStore>& realtime_store,
                              const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
-                             const std::shared_ptr<RealtimeContextImpl>& realtime_context,
-                             const RealtimePartitionBucket& partition_bucket,
                              const std::shared_ptr<arrow::Schema>& write_schema,
-                             int64_t next_offset, const std::shared_ptr<MemoryPool>& memory_pool);
+                             const std::shared_ptr<arrow::Schema>& prepared_schema,
+                             const std::shared_ptr<arrow::Schema>& key_schema,
+                             const std::vector<std::string>& trimmed_primary_keys,
+                             const std::shared_ptr<FieldsComparator>& key_comparator,
+                             int64_t next_offset, int64_t last_sequence_number,
+                             const std::shared_ptr<MemoryPool>& memory_pool);
 
-    Status FlushSegment(const std::shared_ptr<RealtimeSegmentHandle>& segment);
+    Status FlushSegment(const std::shared_ptr<RealtimeSegmentHandle>& segment,
+                        int64_t expected_raw_row_count);
 
     std::shared_ptr<MemoryPool> memory_pool_;
+    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
     std::shared_ptr<RealtimeStore> realtime_store_;
     std::shared_ptr<MergeTreeWriter> merge_tree_writer_;
-    std::shared_ptr<RealtimeContextImpl> realtime_context_;
-    RealtimePartitionBucket partition_bucket_;
     std::shared_ptr<arrow::Schema> write_schema_;
+    std::shared_ptr<arrow::Schema> prepared_schema_;
+    std::shared_ptr<arrow::Schema> key_schema_;
+    std::vector<std::string> trimmed_primary_keys_;
+    std::shared_ptr<FieldsComparator> key_comparator_;
     int64_t next_offset_;
+    int64_t last_sequence_number_;
     std::mutex realtime_store_mutex_;
     std::mutex prepare_mutex_;
 };
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index 76160ac93..f510c987e 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -24,20 +24,21 @@
 
 #include "arrow/api.h"
 #include "arrow/c/bridge.h"
-#include "paimon/common/data/columnar/columnar_row_ref.h"
 #include "paimon/common/reader/concat_batch_reader.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
-#include "paimon/common/types/row_kind.h"
-#include "paimon/common/utils/arrow/arrow_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/global_index/indexed_split_impl.h"
+#include "paimon/core/io/merged_key_value_record_reader.h"
+#include "paimon/core/io/prepared_key_value_reader.h"
 #include "paimon/core/key_value.h"
+#include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
+#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
 #include "paimon/core/operation/merge_file_split_read.h"
 #include "paimon/core/operation/raw_file_split_read.h"
-#include "paimon/core/realtime/primary_key_realtime_store.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
+#include "paimon/core/realtime/realtime_fields.h"
 #include "paimon/core/realtime/realtime_reader.h"
 #include "paimon/core/table/source/data_split_impl.h"
 #include "paimon/core/table/source/pk_count_reader.h"
@@ -54,177 +55,60 @@ struct ColumnarBatchContext;
 
 namespace {
 
-class QueryBatchKeyValueReader final : public KeyValueRecordReader {
- public:
-    QueryBatchKeyValueReader(std::unique_ptr<BatchReader>&& reader,
-                             const std::shared_ptr<arrow::Schema>& key_schema,
-                             const std::shared_ptr<arrow::Schema>& value_schema,
-                             const std::shared_ptr<MemoryPool>& pool)
-        : reader_(std::move(reader)),
-          key_schema_(key_schema),
-          value_schema_(value_schema),
-          pool_(pool) {}
-
-    ~QueryBatchKeyValueReader() override {
-        Close();
-    }
-
-    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatch() override;
-    std::shared_ptr<Metrics> GetReaderMetrics() const override;
-    void Close() override;
-
- private:
-    class Iterator;
-
-    std::unique_ptr<BatchReader> reader_;
-    std::shared_ptr<arrow::Schema> key_schema_;
-    std::shared_ptr<arrow::Schema> value_schema_;
-    std::shared_ptr<MemoryPool> pool_;
-    std::shared_ptr<arrow::StructArray> values_;
-    std::shared_ptr<arrow::Int64Array> sequences_;
-    std::shared_ptr<arrow::Int8Array> row_kinds_;
-    std::shared_ptr<ColumnarBatchContext> key_context_;
-    std::shared_ptr<ColumnarBatchContext> value_context_;
-    bool closed_ = false;
-};
-
-class QueryBatchKeyValueReader::Iterator final : public KeyValueRecordReader::Iterator {
- public:
-    explicit Iterator(QueryBatchKeyValueReader* reader) : reader_(reader) {}
-
-    Result<bool> HasNext() const override {
-        return cursor_ < reader_->values_->length();
-    }
-
-    Result<KeyValue> Next() override {
-        if (reader_->sequences_->IsNull(cursor_) || reader_->row_kinds_->IsNull(cursor_)) {
-            return Status::Invalid("PK merge metadata must not be null");
-        }
-        PAIMON_ASSIGN_OR_RAISE(const RowKind* row_kind,
-                               RowKind::FromByteValue(reader_->row_kinds_->Value(cursor_)));
-        const int64_t sequence = reader_->sequences_->Value(cursor_);
-        std::shared_ptr<InternalRow> key =
-            std::make_shared<ColumnarRowRef>(reader_->key_context_, cursor_);
-        auto value = std::make_unique<ColumnarRowRef>(reader_->value_context_, cursor_++);
-        return KeyValue(row_kind, sequence, KeyValue::UNKNOWN_LEVEL, std::move(key),
-                        std::move(value));
-    }
-
- private:
-    QueryBatchKeyValueReader* reader_;
-    int64_t cursor_ = 0;
-};
-
-Result<std::unique_ptr<KeyValueRecordReader::Iterator>> QueryBatchKeyValueReader::NextBatch() {
-    PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader_->NextBatch());
-    if (BatchReader::IsEofBatch(batch)) {
-        return std::unique_ptr<KeyValueRecordReader::Iterator>();
-    }
-    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> imported,
-                                      arrow::ImportArray(batch.first.get(), batch.second.get()));
-    std::shared_ptr<arrow::StructArray> input =
-        std::dynamic_pointer_cast<arrow::StructArray>(imported);
-    if (!input) {
-        return Status::Invalid("PK merge input is not a StructArray");
-    }
-    sequences_ = std::dynamic_pointer_cast<arrow::Int64Array>(
-        input->GetFieldByName(SpecialFields::SequenceNumber().Name()));
-    row_kinds_ = std::dynamic_pointer_cast<arrow::Int8Array>(
-        input->GetFieldByName(SpecialFields::ValueKind().Name()));
-    if (!sequences_ || !row_kinds_) {
-        return Status::Invalid("PK merge input is missing sequence or value-kind metadata");
-    }
-    PAIMON_ASSIGN_OR_RAISE(input, ArrowUtils::RemoveFieldFromStructArray(
-                                      input, SpecialFields::SequenceNumber().Name()));
-    PAIMON_ASSIGN_OR_RAISE(
-        values_, ArrowUtils::RemoveFieldFromStructArray(input, SpecialFields::ValueKind().Name()));
-    if (!ArrowUtils::EqualsIgnoreNullable(values_->type(),
-                                          arrow::struct_(value_schema_->fields()))) {
-        return Status::Invalid("PK merge input value schema does not match the table read schema");
-    }
-    arrow::ArrayVector key_fields;
-    key_fields.reserve(key_schema_->num_fields());
-    for (const std::shared_ptr<arrow::Field>& field : key_schema_->fields()) {
-        std::shared_ptr<arrow::Array> key = values_->GetFieldByName(field->name());
-        if (!key) {
-            return Status::Invalid("PK merge input is missing key field ", field->name());
-        }
-        key_fields.push_back(std::move(key));
-    }
-    key_context_ = std::make_shared<ColumnarBatchContext>(key_fields, pool_);
-    value_context_ = std::make_shared<ColumnarBatchContext>(values_->fields(), pool_);
-    return std::make_unique<Iterator>(this);
-}
-
-std::shared_ptr<Metrics> QueryBatchKeyValueReader::GetReaderMetrics() const {
-    return reader_->GetReaderMetrics();
-}
-
-void QueryBatchKeyValueReader::Close() {
-    if (closed_) {
-        return;
-    }
-    closed_ = true;
-    values_.reset();
-    sequences_.reset();
-    row_kinds_.reset();
-    key_context_.reset();
-    value_context_.reset();
-    if (reader_) {
-        reader_->Close();
-    }
-}
-
 Result<std::vector<AdditionalKeyValueReader>> CreateMemoryReaders(
     const std::shared_ptr<RealtimeSplit>& split, const RealtimePartitionBucketView& memory,
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
+    const std::shared_ptr<FieldsComparator>& key_comparator,
     const std::shared_ptr<InternalReadContext>& context,
     const std::shared_ptr<MemoryPool>& memory_pool) {
-    arrow::FieldVector requested_fields = {
-        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())};
-    requested_fields.insert(requested_fields.end(), value_schema->fields().begin(),
-                            value_schema->fields().end());
+    std::shared_ptr<arrow::Schema> full_value_schema =
+        DataField::ConvertDataFieldsToArrowSchema(context->GetTableSchema()->Fields());
+    arrow::FieldVector prepared_fields = {
+        DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
+        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
+            ->WithNullable(false),
+        DataField::ConvertDataFieldToArrowField(RealtimeOffsetField())->WithNullable(false)};
+    prepared_fields.insert(prepared_fields.end(), full_value_schema->fields().begin(),
+                           full_value_schema->fields().end());
+    std::shared_ptr<arrow::Schema> prepared_schema = arrow::schema(std::move(prepared_fields));
     auto c_schema = std::make_unique<ArrowSchema>();
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(
-        arrow::ExportSchema(*arrow::schema(requested_fields), c_schema.get()));
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*prepared_schema, c_schema.get()));
     ScopeGuard schema_guard([schema = c_schema.get()]() { ArrowSchemaRelease(schema); });
-    RealtimeQueryContext query_context{c_schema.get(), /*predicate=*/nullptr,
-                                       /*enable_predicate_pushdown=*/false};
-    PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> batch_readers,
-                           memory.store->CreateQueryReaders(
-                               memory.read_view, split->CommittedEndOffset(), query_context));
-    ScopeGuard reader_guard([&batch_readers]() {
+    RealtimeQueryContext query_context{c_schema.get(), nullptr, false};
+    PAIMON_ASSIGN_OR_RAISE(
+        std::vector<std::unique_ptr<BatchReader>> batch_readers,
+        memory.store->CreateQueryReaders(memory.read_view, 0, query_context));
+    ScopeGuard batch_readers_guard([&batch_readers]() {
         for (const std::unique_ptr<BatchReader>& reader : batch_readers) {
             if (reader) {
                 reader->Close();
             }
         }
     });
-    if (batch_readers.empty()) {
-        return Status::Invalid("PK real-time store returned no query readers for active memory");
-    }
     std::vector<AdditionalKeyValueReader> result;
     result.reserve(batch_readers.size());
     for (std::unique_ptr<BatchReader>& reader : batch_readers) {
         if (!reader) {
             return Status::Invalid("PK real-time store returned a null query reader");
         }
-        std::shared_ptr<InternalRow> min_key;
-        std::shared_ptr<InternalRow> max_key;
-        if (auto* provider = dynamic_cast<PrimaryKeyRangeProvider*>(reader.get())) {
-            min_key = provider->GetMinKey();
-            max_key = provider->GetMaxKey();
-        }
-        result.push_back(
-            AdditionalKeyValueReader{std::make_unique<QueryBatchKeyValueReader>(
-                                         std::move(reader), key_schema, value_schema, memory_pool),
-                                     std::move(min_key), std::move(max_key)});
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> prepared_reader,
+                               AdaptPreparedBatchReader(std::move(reader), prepared_schema,
+                                                        OffsetRange(split->CommittedEndOffset(),
+                                                                    split->MemoryEndOffset()),
+                                                        key_schema, value_schema, memory_pool));
+        auto merge = std::make_unique<DeduplicateMergeFunction>(false);
+        result.push_back(AdditionalKeyValueReader{
+            std::make_unique<MergedKeyValueRecordReader>(
+                std::move(prepared_reader), key_comparator,
+                std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge))),
+            nullptr, nullptr});
     }
+    batch_readers_guard.Release();
     return result;
 }
 
-}  // namespace
+}
 
 KeyValueTableRead::KeyValueTableRead(std::vector<std::unique_ptr<SplitRead>>&& split_reads,
                                      const std::shared_ptr<FileStorePathFactory>& path_factory,
@@ -268,7 +152,7 @@ Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateReader(
     const std::shared_ptr<Split>& split) {
     std::shared_ptr<RealtimeSplit> realtime_split = std::dynamic_pointer_cast<RealtimeSplit>(split);
     if (realtime_split) {
-        return CreateRealtimeReader(realtime_split, /*release_ticket=*/true);
+        return CreateRealtimeReader(realtime_split, true);
     }
 
     std::shared_ptr<Split> dispatch_split = split;
@@ -332,7 +216,7 @@ Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateReader(
             std::dynamic_pointer_cast<RealtimeSplit>(split);
         if (realtime_split) {
             PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> reader,
-                                   CreateRealtimeReader(realtime_split, /*release_ticket=*/false));
+                                   CreateRealtimeReader(realtime_split, false));
             readers.push_back(std::move(reader));
             realtime_splits.push_back(std::move(realtime_split));
         } else {
@@ -386,7 +270,8 @@ Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateRealtimeReader(
             PAIMON_ASSIGN_OR_RAISE(
                 std::vector<AdditionalKeyValueReader> memory_readers,
                 CreateMemoryReaders(realtime_split, memory, merge_read->GetKeySchema(),
-                                    merge_read->GetValueSchema(), context_, GetMemoryPool()));
+                                    merge_read->GetValueSchema(), merge_read->GetKeyComparator(),
+                                    context_, GetMemoryPool()));
             PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> reader,
                                    merge_read->CreateRealtimeReader(realtime_split->DiskSplits(),
                                                                     std::move(memory_readers)));
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index 9f302eb37..be7595381 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -443,9 +443,52 @@ class CloseTrackingRealtimeStoreFactory final : public RealtimeStoreFactory {
     std::shared_ptr<CloseTrackingReaderState> state_;
 };
 
-class InvalidReaderRealtimeStore final : public RealtimeStore {
+class SplitBatchReader final : public BatchReader {
  public:
-    explicit InvalidReaderRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate)
+    explicit SplitBatchReader(std::unique_ptr<BatchReader> delegate)
+        : delegate_(std::move(delegate)) {}
+
+    Result<ReadBatch> NextBatch() override {
+        while (!current_batch_ || next_row_ == current_batch_->length()) {
+            PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, delegate_->NextBatch());
+            if (BatchReader::IsEofBatch(batch)) {
+                return MakeEofBatch();
+            }
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+                std::shared_ptr<arrow::Array> array,
+                arrow::ImportArray(batch.first.get(), batch.second.get()));
+            if (!array || array->type_id() != arrow::Type::STRUCT) {
+                return Status::Invalid("split batch reader received a non-struct batch");
+            }
+            current_batch_ = std::dynamic_pointer_cast<arrow::StructArray>(array);
+            next_row_ = 0;
+        }
+        std::shared_ptr<arrow::Array> slice = current_batch_->Slice(next_row_, /*length=*/1);
+        ++next_row_;
+        auto output = std::make_unique<ArrowArray>();
+        auto schema = std::make_unique<ArrowSchema>();
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*slice, output.get(), schema.get()));
+        return ReadBatch(std::move(output), std::move(schema));
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return delegate_->GetReaderMetrics();
+    }
+
+    void Close() override {
+        current_batch_.reset();
+        delegate_->Close();
+    }
+
+ private:
+    std::unique_ptr<BatchReader> delegate_;
+    std::shared_ptr<arrow::StructArray> current_batch_;
+    int64_t next_row_ = 0;
+};
+
+class SplitCommitReaderRealtimeStore final : public RealtimeStore {
+ public:
+    explicit SplitCommitReaderRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate)
         : delegate_(delegate) {}
 
     Status Write(RealtimeWriteBatch&& batch) override {
@@ -457,9 +500,12 @@ class InvalidReaderRealtimeStore final : public RealtimeStore {
     }
 
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
-        const std::shared_ptr<RealtimeSegmentHandle>&) override {
-        std::vector<std::unique_ptr<BatchReader>> readers;
-        readers.push_back(nullptr);
+        const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
+        PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
+                               delegate_->CreateCommitReaders(segment));
+        for (std::unique_ptr<BatchReader>& reader : readers) {
+            reader = std::make_unique<SplitBatchReader>(std::move(reader));
+        }
         return readers;
     }
 
@@ -468,8 +514,9 @@ class InvalidReaderRealtimeStore final : public RealtimeStore {
     }
 
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
-        const std::shared_ptr<RealtimeReadView>&, int64_t, const RealtimeQueryContext&) override {
-        return std::vector<std::unique_ptr<BatchReader>>();
+        const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
+        const RealtimeQueryContext& context) override {
+        return delegate_->CreateQueryReaders(view, offset_begin, context);
     }
 
     Status AdvanceCommittedOffset(int64_t committed_offset) override {
@@ -484,13 +531,13 @@ class InvalidReaderRealtimeStore final : public RealtimeStore {
     std::shared_ptr<RealtimeStore> delegate_;
 };
 
-class InvalidReaderRealtimeStoreFactory final : public RealtimeStoreFactory {
+class SplitCommitReaderRealtimeStoreFactory final : public RealtimeStoreFactory {
  public:
     Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> delegate,
                                delegate_.Create(std::move(request)));
         return std::shared_ptr<RealtimeStore>(
-            std::make_shared<InvalidReaderRealtimeStore>(delegate));
+            std::make_shared<SplitCommitReaderRealtimeStore>(delegate));
     }
 
  private:
@@ -1311,10 +1358,11 @@ TEST_F(RealtimeWriteInteTest, TestPkRead) {
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                          CreateRealtimeWriter(realtime_context));
 
-    std::vector<Row> first_rows = {{1, "old", "p0"}, {2, "two", "p0"}};
+    std::vector<Row> first_rows = {{1, "old", "p0"}, {2, "two", "p0"}, {1, "new-in-run", "p0"}};
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
                          MakeBatch(first_rows, /*partitioned=*/false, /*bucket=*/0,
-                                   {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::INSERT}));
+                                   {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::INSERT,
+                                    RecordBatch::RowKind::UPDATE_AFTER}));
     ASSERT_OK(writer->Write(std::move(first_batch)));
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> update_batch,
                          MakeBatch({Row{1, "new", "p0"}}, /*partitioned=*/false, /*bucket=*/0,
@@ -1756,13 +1804,13 @@ TEST_F(RealtimeWriteInteTest, TestPkRecovery) {
                          MakeBatch(mutations, /*partitioned=*/false, /*bucket=*/0, mutation_kinds));
     ASSERT_OK(first_writer->Write(std::move(batch)));
     ASSERT_OK_AND_ASSIGN(std::vector<int64_t> memory_sequences, ReadPkSequences(first_context));
-    ASSERT_EQ((std::vector<int64_t>{2, 3, 4}), memory_sequences);
+    ASSERT_EQ((std::vector<int64_t>{1, 2, 3, 4}), memory_sequences);
     ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                          first_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
     ASSERT_EQ(1, progress.size());
     ASSERT_EQ(OffsetRange(0, 4), progress[0].offset_range);
     ASSERT_EQ(1, NewFiles(progress).size());
-    ASSERT_EQ(memory_sequences.front(), NewFiles(progress)[0]->min_sequence_number);
+    ASSERT_EQ(2, NewFiles(progress)[0]->min_sequence_number);
     ASSERT_EQ(memory_sequences.back(), NewFiles(progress)[0]->max_sequence_number);
     ASSERT_OK(Commit(progress, /*commit_identifier=*/1));
     ASSERT_OK(first_writer->Close());
@@ -1801,9 +1849,16 @@ TEST_F(RealtimeWriteInteTest, TestPkCompaction) {
     constexpr int64_t kCommitRoundsBeforeCompaction = 4;
     std::set<std::string> committed_file_names;
     for (int64_t round = 0; round < kCommitRoundsBeforeCompaction; ++round) {
-        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
-                             MakeBatch({Row{round, "value-" + std::to_string(round), "p0"}},
-                                       /*partitioned=*/false));
+        const bool delete_latest_live_row = round == kCommitRoundsBeforeCompaction - 1;
+        ASSERT_OK_AND_ASSIGN(
+            std::unique_ptr<RecordBatch> batch,
+            MakeBatch(
+                {Row{delete_latest_live_row ? round - 1 : round,
+                     delete_latest_live_row ? "deleted" : "value-" + std::to_string(round), "p0"}},
+                /*partitioned=*/false, /*bucket=*/0,
+                delete_latest_live_row
+                    ? std::vector<RecordBatch::RowKind>{RecordBatch::RowKind::DELETE}
+                    : std::vector<RecordBatch::RowKind>{}));
         ASSERT_OK(writer->Write(std::move(batch)));
         ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                              writer->PrepareCommitWithProgress(round));
@@ -1819,11 +1874,6 @@ TEST_F(RealtimeWriteInteTest, TestPkCompaction) {
         ASSERT_OK_AND_ASSIGN(uint64_t memory_usage, GetRealtimeMemoryUsage(realtime_context));
         ASSERT_EQ(0, memory_usage);
     }
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> next_batch,
-                         MakeBatch({Row{4, "value-4", "p0"}},
-                                   /*partitioned=*/false));
-    ASSERT_OK(writer->Write(std::move(next_batch)));
-
     WriteContextBuilder compact_builder(table_path_, commit_user_);
     compact_builder.SetOptions(options_).WithStreamingMode(true);
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> compact_context, compact_builder.Finish());
@@ -1848,6 +1898,14 @@ TEST_F(RealtimeWriteInteTest, TestPkCompaction) {
     }
     ASSERT_EQ(committed_file_names, compacted_file_names);
     ASSERT_FALSE(compact_message->GetCompactIncrement().CompactAfter().empty());
+    constexpr int64_t kHistoricalMaxSequenceNumber = kCommitRoundsBeforeCompaction - 1;
+    int64_t compacted_live_max_sequence_number = -1;
+    for (const std::shared_ptr<DataFileMeta>& file :
+         compact_message->GetCompactIncrement().CompactAfter()) {
+        compacted_live_max_sequence_number =
+            std::max(compacted_live_max_sequence_number, file->max_sequence_number);
+    }
+    ASSERT_LT(compacted_live_max_sequence_number, kHistoricalMaxSequenceNumber);
     ASSERT_OK(CommitMessages(compact_messages, /*commit_identifier=*/4));
     ASSERT_OK(compact_writer->Close());
 
@@ -1859,45 +1917,38 @@ TEST_F(RealtimeWriteInteTest, TestPkCompaction) {
     ASSERT_EQ(Snapshot::CommitKind::Compact(), compact_snapshot->GetCommitKind());
     ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap offsets, ReadCommittedOffsets());
     ASSERT_EQ(4, offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
-    ASSERT_OK(writer->RefreshCommittedSnapshot(compact_snapshot->Id()));
-    ASSERT_OK_AND_ASSIGN(std::vector<Row> compacted_rows, ReadRows(realtime_context));
-    ASSERT_EQ((std::vector<Row>{{0, "value-0", "p0"},
-                                {1, "value-1", "p0"},
-                                {2, "value-2", "p0"},
-                                {3, "value-3", "p0"},
-                                {4, "value-4", "p0"}}),
-              compacted_rows);
-
-    constexpr int64_t kCommitRoundsAfterCompaction = 2;
-    for (int64_t round = 0; round < kCommitRoundsAfterCompaction; ++round) {
-        if (round > 0) {
-            ASSERT_OK_AND_ASSIGN(
-                std::unique_ptr<RecordBatch> batch,
-                MakeBatch({Row{4 + round, "value-" + std::to_string(4 + round), "p0"}},
-                          /*partitioned=*/false));
-            ASSERT_OK(writer->Write(std::move(batch)));
-        }
-        const int64_t commit_identifier = 5 + round;
-        ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
-                             writer->PrepareCommitWithProgress(commit_identifier));
-        ASSERT_EQ(1, progress.size());
-        ASSERT_EQ(OffsetRange(4 + round, 5 + round), progress[0].offset_range);
-        ASSERT_OK_AND_ASSIGN(latest_snapshot_id, Commit(progress, commit_identifier));
-        ASSERT_OK(writer->RefreshCommittedSnapshot(latest_snapshot_id));
-        ASSERT_OK_AND_ASSIGN(uint64_t memory_usage, GetRealtimeMemoryUsage(realtime_context));
-        ASSERT_EQ(0, memory_usage);
-    }
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> compacted_rows, ReadRows());
+    ASSERT_EQ((std::vector<Row>{{0, "value-0", "p0"}, {1, "value-1", "p0"}}), compacted_rows);
+    ASSERT_OK(writer->Close());
+    writer.reset();
+    realtime_context.reset();
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> fresh_context, RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> fresh_writer,
+                         CreateRealtimeWriter(fresh_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> fresh_batch,
+                         MakeBatch({Row{4, "value-4", "p0"}},
+                                   /*partitioned=*/false));
+    ASSERT_OK(fresh_writer->Write(std::move(fresh_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<int64_t> fresh_sequences, ReadPkSequences(fresh_context));
+    ASSERT_EQ((std::vector<int64_t>{compacted_live_max_sequence_number + 1}), fresh_sequences);
+    ASSERT_LT(fresh_sequences.front(), kHistoricalMaxSequenceNumber);
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> fresh_progress,
+                         fresh_writer->PrepareCommitWithProgress(/*commit_identifier=*/5));
+    ASSERT_EQ(1, fresh_progress.size());
+    ASSERT_EQ(OffsetRange(4, 5), fresh_progress[0].offset_range);
+    ASSERT_EQ(compacted_live_max_sequence_number + 1,
+              NewFiles(fresh_progress)[0]->min_sequence_number);
+    ASSERT_EQ(compacted_live_max_sequence_number + 1,
+              NewFiles(fresh_progress)[0]->max_sequence_number);
+    ASSERT_OK_AND_ASSIGN(latest_snapshot_id, Commit(fresh_progress, /*commit_identifier=*/5));
+    ASSERT_OK(fresh_writer->Close());
+
     ASSERT_OK_AND_ASSIGN(offsets, ReadCommittedOffsets());
-    ASSERT_EQ(6, offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
-    ASSERT_OK_AND_ASSIGN(std::vector<Row> final_rows, ReadRows(realtime_context));
-    ASSERT_EQ((std::vector<Row>{{0, "value-0", "p0"},
-                                {1, "value-1", "p0"},
-                                {2, "value-2", "p0"},
-                                {3, "value-3", "p0"},
-                                {4, "value-4", "p0"},
-                                {5, "value-5", "p0"}}),
+    ASSERT_EQ(5, offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> final_rows, ReadRows());
+    ASSERT_EQ((std::vector<Row>{{0, "value-0", "p0"}, {1, "value-1", "p0"}, {4, "value-4", "p0"}}),
               final_rows);
-    ASSERT_OK(writer->Close());
 }
 
 TEST_F(RealtimeWriteInteTest, TestPkConcurrency) {
@@ -2054,19 +2105,29 @@ TEST_F(RealtimeWriteInteTest, TestPkWriteDuringPrepare) {
 
 TEST_F(RealtimeWriteInteTest, TestPkPluginContract) {
     CreatePkTable();
-    auto factory = std::make_shared<InvalidReaderRealtimeStoreFactory>();
+    auto factory = std::make_shared<SplitCommitReaderRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                          RealtimeContext::Create(factory));
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                          CreateRealtimeWriter(realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
-                         MakeBatch({Row{1, "one", "p0"}}, /*partitioned=*/false));
-    ASSERT_OK(writer->Write(std::move(batch)));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
+                         MakeBatch({Row{4, "four", "p0"}, Row{3, "three", "p0"}},
+                                   /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(first_batch)));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
+                         MakeBatch({Row{2, "two", "p0"}, Row{1, "one", "p0"}},
+                                   /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(second_batch)));
 
-    ASSERT_NOK_WITH_MSG(ReadRows(realtime_context),
-                        "PK real-time store returned no query readers for active memory");
-    ASSERT_NOK_WITH_MSG(writer->PrepareCommitWithProgress(/*commit_identifier=*/0),
-                        "PK real-time store returned a null commit reader");
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_EQ(1, progress.size());
+    ASSERT_EQ(OffsetRange(0, 4), progress[0].offset_range);
+    ASSERT_OK(Commit(progress, /*commit_identifier=*/0));
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows, ReadRows());
+    ASSERT_EQ((std::vector<Row>{
+                  {1, "one", "p0"}, {2, "two", "p0"}, {3, "three", "p0"}, {4, "four", "p0"}}),
+              rows);
     ASSERT_OK(writer->Close());
 }
 
@@ -2757,52 +2818,6 @@ TEST_F(RealtimeWriteInteTest, TestCloseWriterAllowsContextReuseByLaterWriter) {
     ASSERT_OK(second_writer->Close());
 }
 
-TEST_F(RealtimeWriteInteTest, TestPkWriterHandoff) {
-    CreatePkTable();
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> first_writer,
-                         CreateRealtimeWriter(realtime_context));
-    std::vector<Row> first_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
-                         MakeBatch(first_rows, /*partitioned=*/false));
-    ASSERT_OK(first_writer->Write(std::move(first_batch)));
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> commits,
-                         first_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
-    ASSERT_EQ(1, commits.size());
-    ASSERT_EQ(OffsetRange(0, 3), commits[0].offset_range);
-    ASSERT_EQ(1, NewFiles(commits).size());
-    ASSERT_EQ(0, NewFiles(commits)[0]->min_sequence_number);
-    ASSERT_EQ(2, NewFiles(commits)[0]->max_sequence_number);
-    ASSERT_OK(first_writer->Close());
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> second_writer,
-                         CreateRealtimeWriter(realtime_context));
-    std::vector<Row> second_rows = {
-        Row{0, "updated-0", "p0"},
-        Row{3, "value-3", "p0"},
-    };
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
-                         MakeBatch(second_rows, /*partitioned=*/false));
-    ASSERT_OK(second_writer->Write(std::move(second_batch)));
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_commits,
-                         second_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
-    ASSERT_EQ(1, second_commits.size());
-    ASSERT_EQ(OffsetRange(3, 5), second_commits[0].offset_range);
-    ASSERT_EQ(1, NewFiles(second_commits).size());
-    ASSERT_EQ(3, NewFiles(second_commits)[0]->min_sequence_number);
-    ASSERT_EQ(4, NewFiles(second_commits)[0]->max_sequence_number);
-
-    commits.push_back(std::move(second_commits[0]));
-    ASSERT_OK(Commit(commits, /*commit_identifier=*/1));
-    std::vector<Row> expected_rows = first_rows;
-    expected_rows[0] = second_rows[0];
-    expected_rows.push_back(second_rows[1]);
-    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(realtime_context));
-    ASSERT_EQ(expected_rows, actual_rows);
-    ASSERT_OK(second_writer->Close());
-}
-
 TEST_F(RealtimeWriteInteTest, TestReadCommittedDiskAndBuildingMemory) {
     CreateTable(/*partition_keys=*/{});
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,

From 62108e2456407e0bbb2012d5aa5343018378e9b0 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Mon, 24 Aug 2026 12:35:37 +0800
Subject: [PATCH 16/62] refactor(realtime): simplify primary-key write
 preparation

---
 include/paimon/realtime/realtime_context.h    |   4 -
 src/paimon/CMakeLists.txt                     |   2 +-
 .../merged_key_value_record_reader_test.cpp   |   2 +-
 .../core/mergetree/merge_tree_writer_test.cpp |  15 -
 src/paimon/core/mergetree/write_buffer.cpp    |   4 -
 .../key_value_file_store_write_test.cpp       |  98 -------
 .../prepared_key_value_reader.cpp             |   2 +-
 .../prepared_key_value_reader.h               |   0
 .../realtime/realtime_primary_key_writer.cpp  |   2 +-
 .../table/source/key_value_table_read.cpp     |   2 +-
 .../core/utils/primary_key_table_utils.h      |   1 -
 test/inte/realtime_write_inte_test.cpp        | 273 ------------------
 12 files changed, 5 insertions(+), 400 deletions(-)
 rename src/paimon/core/{io => realtime}/prepared_key_value_reader.cpp (99%)
 rename src/paimon/core/{io => realtime}/prepared_key_value_reader.h (100%)

diff --git a/include/paimon/realtime/realtime_context.h b/include/paimon/realtime/realtime_context.h
index 8f2967b32..200e4ba4c 100644
--- a/include/paimon/realtime/realtime_context.h
+++ b/include/paimon/realtime/realtime_context.h
@@ -78,10 +78,6 @@ using RealtimeOffsetMap = std::map<RealtimePartitionBucket, int64_t>;
 /// partition drop, and rollback operations do not automatically clear process-local real-time
 /// state. Applications must coordinate these operations with active real-time writers and recreate
 /// the `RealtimeContext` and writers before continuing.
-///
-/// A primary-key writer and its context form one lifecycle. After a primary-key write or prepare
-/// returns an error, discard both, create fresh instances from the latest committed snapshot, and
-/// replay the caller-owned WAL. Reusing the failed context or retrying prepare is unsupported.
 class PAIMON_EXPORT RealtimeContext {
  public:
     /// Creates a context backed by Paimon's default in-memory Arrow `RealtimeStore`.
diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index 62af55ec4..0a78b0902 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -286,7 +286,6 @@ set(PAIMON_CORE_SRCS
     core/io/vector_file_batch_reader.cpp
     core/io/file_index_evaluator.cpp
     core/io/key_value_data_file_record_reader.cpp
-    core/io/prepared_key_value_reader.cpp
     core/io/key_value_data_file_writer_factory.cpp
     core/io/key_value_data_file_writer.cpp
     core/io/key_value_in_memory_record_reader.cpp
@@ -383,6 +382,7 @@ set(PAIMON_CORE_SRCS
     core/operation/write_restore.cpp
     core/realtime/arrow_realtime_store.cpp
     core/realtime/arrow_realtime_store_factory.cpp
+    core/realtime/prepared_key_value_reader.cpp
     core/realtime/primary_key_realtime_store.cpp
     core/realtime/realtime_append_only_writer.cpp
     core/realtime/realtime_context.cpp
diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index 39714fa29..21b0a16b1 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -34,9 +34,9 @@
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/fields_comparator.h"
-#include "paimon/core/io/prepared_key_value_reader.h"
 #include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
 #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
+#include "paimon/core/realtime/prepared_key_value_reader.h"
 #include "paimon/core/realtime/realtime_fields.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/realtime/offset_range.h"
diff --git a/src/paimon/core/mergetree/merge_tree_writer_test.cpp b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
index aa2d0c959..675ce3198 100644
--- a/src/paimon/core/mergetree/merge_tree_writer_test.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
@@ -21,7 +21,6 @@
 #include <cassert>
 #include <cstddef>
 #include <functional>
-#include <limits>
 #include <map>
 #include <optional>
 #include <utility>
@@ -613,20 +612,6 @@ TEST_P(MergeTreeWriterTest, TestSortedReaderFailure) {
     ASSERT_OK(merge_writer->Close());
 }
 
-TEST_P(MergeTreeWriterTest, TestRejectsExhaustedSequence) {
-    ASSERT_OK_AND_ASSIGN(CoreOptions options,
-                         CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}}));
-
-    auto dir = UniqueTestDirectory::Create();
-    ASSERT_TRUE(dir);
-    auto path_factory = std::make_shared<DataFilePathFactory>();
-    ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr));
-
-    ASSERT_NOK_WITH_MSG(CreateMergeWriter(std::numeric_limits<int64_t>::max(), dir->Str(),
-                                          path_factory, 0, options),
-                        "sequence number has reached INT64_MAX");
-}
-
 TEST_P(MergeTreeWriterTest, TestSharedShreddingMapDataFileMetaInfo) {
     ASSERT_OK_AND_ASSIGN(CoreOptions options,
                          CoreOptions::FromMap({
diff --git a/src/paimon/core/mergetree/write_buffer.cpp b/src/paimon/core/mergetree/write_buffer.cpp
index 3d3fdc196..549975a33 100644
--- a/src/paimon/core/mergetree/write_buffer.cpp
+++ b/src/paimon/core/mergetree/write_buffer.cpp
@@ -18,7 +18,6 @@
 
 #include "paimon/core/mergetree/write_buffer.h"
 
-#include <limits>
 #include <memory>
 #include <utility>
 
@@ -40,9 +39,6 @@ Result<std::unique_ptr<WriteBuffer>> WriteBuffer::Create(
     const std::shared_ptr<MergeFunctionWrapper<KeyValue>>& merge_function_wrapper,
     const CoreOptions& options, const std::shared_ptr<IOManager>& io_manager,
     bool enable_multi_thread_spill, const std::shared_ptr<MemoryPool>& pool) {
-    if (last_sequence_number == std::numeric_limits<int64_t>::max()) {
-        return Status::Invalid("sequence number has reached INT64_MAX");
-    }
     auto value_type = arrow::struct_(value_schema->fields());
     auto in_memory_buffer = std::make_unique<InMemorySortBuffer>(
         last_sequence_number, value_type, trimmed_primary_keys, user_defined_sequence_fields,
diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp
index cbd2189fc..733c19d6e 100644
--- a/src/paimon/core/operation/key_value_file_store_write_test.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp
@@ -59,7 +59,6 @@
 #include "paimon/fs/local/local_file_system.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/reader/file_batch_reader.h"
-#include "paimon/realtime/arrow_realtime_store_factory.h"
 #include "paimon/realtime/realtime_context.h"
 #include "paimon/record_batch.h"
 #include "paimon/status.h"
@@ -111,69 +110,6 @@ class TestingMemoryPool final : public MemoryPool {
     std::unique_ptr<MemoryPool> delegate_ = GetMemoryPool();
 };
 
-class FailOnceRealtimeStore final : public RealtimeStore {
- public:
-    FailOnceRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
-                          const std::shared_ptr<bool>& fail_next_write)
-        : delegate_(delegate), fail_next_write_(fail_next_write) {}
-
-    Status Write(RealtimeWriteBatch&& batch) override {
-        if (*fail_next_write_) {
-            *fail_next_write_ = false;
-            return Status::Invalid("injected real-time store write failure");
-        }
-        return delegate_->Write(std::move(batch));
-    }
-
-    Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>> SealForCommit() override {
-        return delegate_->SealForCommit();
-    }
-
-    Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
-        const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
-        return delegate_->CreateCommitReaders(segment);
-    }
-
-    Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
-        return delegate_->AcquireReadView();
-    }
-
-    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
-        const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
-        const RealtimeQueryContext& context) override {
-        return delegate_->CreateQueryReaders(view, offset_begin, context);
-    }
-
-    Status AdvanceCommittedOffset(int64_t committed_offset) override {
-        return delegate_->AdvanceCommittedOffset(committed_offset);
-    }
-
-    uint64_t GetMemoryUsage() const override {
-        return delegate_->GetMemoryUsage();
-    }
-
- private:
-    std::shared_ptr<RealtimeStore> delegate_;
-    std::shared_ptr<bool> fail_next_write_;
-};
-
-class FailOnceRealtimeStoreFactory final : public RealtimeStoreFactory {
- public:
-    explicit FailOnceRealtimeStoreFactory(const std::shared_ptr<bool>& fail_next_write)
-        : fail_next_write_(fail_next_write) {}
-
-    Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> delegate,
-                               delegate_.Create(std::move(request)));
-        return std::shared_ptr<RealtimeStore>(
-            std::make_shared<FailOnceRealtimeStore>(delegate, fail_next_write_));
-    }
-
- private:
-    ArrowRealtimeStoreFactory delegate_;
-    std::shared_ptr<bool> fail_next_write_;
-};
-
 }
 
 class KeyValueFileStoreWriteTest : public ::testing::Test {
@@ -551,40 +487,6 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeOffsetCollision) {
     ASSERT_OK(writer->Close());
 }
 
-TEST_F(KeyValueFileStoreWriteTest, TestWriteFailureKeepsCursors) {
-    const std::map<std::string, std::string> options = {
-        {Options::BUCKET, "1"},
-        {Options::WRITE_BUFFER_SIZE, "1"},
-    };
-    const std::shared_ptr<arrow::Schema> schema = arrow::schema({
-        arrow::field("id", arrow::int64(), false),
-        arrow::field("value", arrow::utf8()),
-    });
-    std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
-    ASSERT_TRUE(dir);
-    CreateTable(dir->Str(), schema, options);
-    const std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar");
-
-    auto fail_next_write = std::make_shared<bool>(true);
-    auto factory = std::make_shared<FailOnceRealtimeStoreFactory>(fail_next_write);
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create(factory));
-    WriteContextBuilder builder(table_path, "test");
-    builder.SetOptions(options).WithStreamingMode(true).WithRealtimeContext(realtime_context);
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> write_context, builder.Finish());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                         FileStoreWrite::Create(std::move(write_context)));
-
-    ASSERT_NOK_WITH_MSG(writer->Write(MakeBatch(schema, R"([[9, "rejected"]])")),
-                        "injected real-time store write failure");
-    ASSERT_OK(writer->Write(MakeBatch(schema, R"([[1, "kept"]])")));
-    using PreparedRow = std::tuple<int8_t, int64_t, std::string, int64_t, int64_t>;
-    ASSERT_OK_AND_ASSIGN(std::vector<PreparedRow> prepared_rows,
-                         ReadPreparedRows(realtime_context));
-    ASSERT_EQ((std::vector<PreparedRow>{{0, 1, "kept", 0, 0}}), prepared_rows);
-    ASSERT_OK(writer->Close());
-}
-
 TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) {
     const std::map<std::string, std::string> options = {{Options::BUCKET, "1"}};
     const std::shared_ptr<arrow::Schema> schema = arrow::schema({
diff --git a/src/paimon/core/io/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
similarity index 99%
rename from src/paimon/core/io/prepared_key_value_reader.cpp
rename to src/paimon/core/realtime/prepared_key_value_reader.cpp
index 0f4f22097..b99f67dd9 100644
--- a/src/paimon/core/io/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-#include "paimon/core/io/prepared_key_value_reader.h"
+#include "paimon/core/realtime/prepared_key_value_reader.h"
 
 #include <cstdint>
 #include <memory>
diff --git a/src/paimon/core/io/prepared_key_value_reader.h b/src/paimon/core/realtime/prepared_key_value_reader.h
similarity index 100%
rename from src/paimon/core/io/prepared_key_value_reader.h
rename to src/paimon/core/realtime/prepared_key_value_reader.h
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index c85ff6322..2dc5a71b4 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -34,10 +34,10 @@
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/io/merged_key_value_record_reader.h"
-#include "paimon/core/io/prepared_key_value_reader.h"
 #include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
 #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
 #include "paimon/core/mergetree/merge_tree_writer.h"
+#include "paimon/core/realtime/prepared_key_value_reader.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
 #include "paimon/core/realtime/realtime_fields.h"
 #include "paimon/core/utils/commit_increment.h"
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index f510c987e..31779e049 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -31,12 +31,12 @@
 #include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/global_index/indexed_split_impl.h"
 #include "paimon/core/io/merged_key_value_record_reader.h"
-#include "paimon/core/io/prepared_key_value_reader.h"
 #include "paimon/core/key_value.h"
 #include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
 #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
 #include "paimon/core/operation/merge_file_split_read.h"
 #include "paimon/core/operation/raw_file_split_read.h"
+#include "paimon/core/realtime/prepared_key_value_reader.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
 #include "paimon/core/realtime/realtime_fields.h"
 #include "paimon/core/realtime/realtime_reader.h"
diff --git a/src/paimon/core/utils/primary_key_table_utils.h b/src/paimon/core/utils/primary_key_table_utils.h
index c40e92cda..82a108ab7 100644
--- a/src/paimon/core/utils/primary_key_table_utils.h
+++ b/src/paimon/core/utils/primary_key_table_utils.h
@@ -24,7 +24,6 @@
 
 #include "arrow/type.h"
 #include "paimon/result.h"
-#include "paimon/status.h"
 
 namespace arrow {
 class Schema;
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index be7595381..c61b04b0d 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -23,7 +23,6 @@
 #include <condition_variable>
 #include <cstdint>
 #include <deque>
-#include <future>
 #include <iterator>
 #include <limits>
 #include <map>
@@ -79,126 +78,6 @@
 namespace paimon::test {
 namespace {
 
-class BlockingState {
- public:
-    void Block() {
-        std::unique_lock<std::mutex> lock(mutex_);
-        entered_ = true;
-        entered_cv_.notify_all();
-        release_cv_.wait(lock, [this]() { return released_; });
-    }
-
-    bool WaitUntilBlocked() {
-        std::unique_lock<std::mutex> lock(mutex_);
-        return entered_cv_.wait_for(lock, std::chrono::seconds(30), [this]() { return entered_; });
-    }
-
-    void Release() {
-        std::lock_guard<std::mutex> lock(mutex_);
-        released_ = true;
-        release_cv_.notify_all();
-    }
-
- private:
-    std::mutex mutex_;
-    std::condition_variable entered_cv_;
-    std::condition_variable release_cv_;
-    bool entered_ = false;
-    bool released_ = false;
-};
-
-class BlockingBatchReader final : public BatchReader {
- public:
-    BlockingBatchReader(std::unique_ptr<BatchReader>&& reader,
-                        const std::shared_ptr<BlockingState>& state)
-        : reader_(std::move(reader)), state_(state) {}
-
-    Result<ReadBatch> NextBatch() override {
-        if (!blocked_) {
-            blocked_ = true;
-            state_->Block();
-        }
-        return reader_->NextBatch();
-    }
-
-    std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return reader_->GetReaderMetrics();
-    }
-
-    void Close() override {
-        reader_->Close();
-    }
-
- private:
-    std::unique_ptr<BatchReader> reader_;
-    std::shared_ptr<BlockingState> state_;
-    bool blocked_ = false;
-};
-
-class BlockingRealtimeStore final : public RealtimeStore {
- public:
-    BlockingRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
-                          const std::shared_ptr<BlockingState>& state)
-        : delegate_(delegate), state_(state) {}
-
-    Status Write(RealtimeWriteBatch&& batch) override {
-        return delegate_->Write(std::move(batch));
-    }
-
-    Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>> SealForCommit() override {
-        return delegate_->SealForCommit();
-    }
-
-    Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
-        const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
-        PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
-                               delegate_->CreateCommitReaders(segment));
-        if (!readers.empty()) {
-            readers[0] = std::make_unique<BlockingBatchReader>(std::move(readers[0]), state_);
-        }
-        return readers;
-    }
-
-    Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
-        return delegate_->AcquireReadView();
-    }
-
-    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
-        const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
-        const RealtimeQueryContext& context) override {
-        return delegate_->CreateQueryReaders(view, offset_begin, context);
-    }
-
-    Status AdvanceCommittedOffset(int64_t committed_offset) override {
-        return delegate_->AdvanceCommittedOffset(committed_offset);
-    }
-
-    uint64_t GetMemoryUsage() const override {
-        return delegate_->GetMemoryUsage();
-    }
-
- private:
-    std::shared_ptr<RealtimeStore> delegate_;
-    std::shared_ptr<BlockingState> state_;
-};
-
-class BlockingRealtimeStoreFactory final : public RealtimeStoreFactory {
- public:
-    explicit BlockingRealtimeStoreFactory(const std::shared_ptr<BlockingState>& state)
-        : state_(state) {}
-
-    Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> delegate,
-                               delegate_.Create(std::move(request)));
-        return std::shared_ptr<RealtimeStore>(
-            std::make_shared<BlockingRealtimeStore>(delegate, state_));
-    }
-
- private:
-    ArrowRealtimeStoreFactory delegate_;
-    std::shared_ptr<BlockingState> state_;
-};
-
 class TrackingRealtimeReadView final : public RealtimeReadView {
  public:
     explicit TrackingRealtimeReadView(std::shared_ptr<RealtimeReadView> delegate)
@@ -1951,158 +1830,6 @@ TEST_F(RealtimeWriteInteTest, TestPkCompaction) {
               final_rows);
 }
 
-TEST_F(RealtimeWriteInteTest, TestPkConcurrency) {
-    CreatePkTable();
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                         CreateRealtimeWriter(realtime_context));
-
-    constexpr int64_t kRowCount = 20;
-    constexpr int32_t kReaderCount = 2;
-    std::atomic<bool> writer_done{false};
-    std::atomic<bool> control_done{false};
-    std::atomic<int32_t> commit_count{0};
-    ConcurrentTestState state;
-    std::vector<int32_t> read_counts(kReaderCount, 0);
-
-    std::thread write_thread([&]() {
-        state.WaitForStart();
-        for (int64_t id = 0; id < kRowCount && !state.ShouldStop(); ++id) {
-            Result<std::unique_ptr<RecordBatch>> batch =
-                MakeBatch(MakeRows(id, /*count=*/1, /*partition=*/"p0"),
-                          /*partitioned=*/false);
-            if (state.RecordErrorIfNotOk(batch) ||
-                state.RecordErrorIfNotOk(writer->Write(std::move(batch).value()))) {
-                break;
-            }
-            std::this_thread::sleep_for(std::chrono::milliseconds(2));
-        }
-        writer_done.store(true, std::memory_order_release);
-    });
-
-    std::thread control_thread([&]() {
-        state.WaitForStart();
-        int64_t commit_identifier = 0;
-        do {
-            Result<std::vector<RealtimeCommitProgress>> progress =
-                writer->PrepareCommitWithProgress(commit_identifier);
-            if (state.RecordErrorIfNotOk(progress)) {
-                break;
-            }
-            if (!progress.value().empty()) {
-                Result<int64_t> snapshot = Commit(progress.value(), commit_identifier++);
-                if (state.RecordErrorIfNotOk(snapshot) ||
-                    state.RecordErrorIfNotOk(writer->RefreshCommittedSnapshot(snapshot.value()))) {
-                    break;
-                }
-                commit_count.fetch_add(1, std::memory_order_relaxed);
-            }
-            std::this_thread::sleep_for(std::chrono::milliseconds(1));
-        } while (!writer_done.load(std::memory_order_acquire) && !state.ShouldStop());
-        if (!state.ShouldStop()) {
-            Result<std::vector<RealtimeCommitProgress>> progress =
-                writer->PrepareCommitWithProgress(commit_identifier);
-            if (!state.RecordErrorIfNotOk(progress) && !progress.value().empty()) {
-                Result<int64_t> snapshot = Commit(progress.value(), commit_identifier);
-                if (!state.RecordErrorIfNotOk(snapshot) &&
-                    !state.RecordErrorIfNotOk(writer->RefreshCommittedSnapshot(snapshot.value()))) {
-                    commit_count.fetch_add(1, std::memory_order_relaxed);
-                }
-            }
-        }
-        control_done.store(true, std::memory_order_release);
-    });
-
-    std::vector<std::thread> read_threads;
-    read_threads.reserve(kReaderCount);
-    for (int32_t reader_index = 0; reader_index < kReaderCount; ++reader_index) {
-        read_threads.emplace_back([&, reader_index]() {
-            state.WaitForStart();
-            while (!control_done.load(std::memory_order_acquire) && !state.ShouldStop()) {
-                Result<std::vector<Row>> rows = ReadRows(realtime_context);
-                ++read_counts[reader_index];
-                if (state.RecordErrorIfNotOk(rows) ||
-                    state.RecordErrorIfNotOk(ValidateReadPrefix(rows.value(), kRowCount))) {
-                    break;
-                }
-                std::this_thread::sleep_for(std::chrono::milliseconds(1));
-            }
-        });
-    }
-
-    state.StartWhenReady(/*worker_count=*/2 + kReaderCount);
-    write_thread.join();
-    control_thread.join();
-    for (std::thread& read_thread : read_threads) {
-        read_thread.join();
-    }
-
-    ASSERT_TRUE(state.Errors().empty()) << (state.Errors().empty() ? "" : state.Errors().front());
-    ASSERT_GT(commit_count.load(), 0);
-    for (int32_t read_count : read_counts) {
-        ASSERT_GT(read_count, 0);
-    }
-    ASSERT_OK_AND_ASSIGN(std::vector<Row> final_rows, ReadRows(realtime_context));
-    ASSERT_EQ(kRowCount, static_cast<int64_t>(final_rows.size()));
-    ASSERT_OK(ValidateReadPrefix(final_rows, kRowCount));
-    ASSERT_OK_AND_ASSIGN(uint64_t memory_usage, GetRealtimeMemoryUsage(realtime_context));
-    ASSERT_EQ(0, memory_usage);
-    ASSERT_OK(writer->Close());
-}
-
-TEST_F(RealtimeWriteInteTest, TestPkWriteDuringPrepare) {
-    CreatePkTable();
-    auto state = std::make_shared<BlockingState>();
-    auto factory = std::make_shared<BlockingRealtimeStoreFactory>(state);
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create(factory));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                         CreateRealtimeWriter(realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
-                         MakeBatch({Row{1, "one", "p0"}}, /*partitioned=*/false));
-    ASSERT_OK(writer->Write(std::move(first_batch)));
-
-    Result<std::vector<RealtimeCommitProgress>> prepare_result =
-        Status::Invalid("prepare did not run");
-    std::thread prepare_thread(
-        [&]() { prepare_result = writer->PrepareCommitWithProgress(/*commit_identifier=*/0); });
-    const bool prepare_blocked = state->WaitUntilBlocked();
-    if (!prepare_blocked) {
-        state->Release();
-        prepare_thread.join();
-        ASSERT_TRUE(prepare_blocked);
-    }
-
-    std::promise<Status> write_promise;
-    std::future<Status> write_future = write_promise.get_future();
-    std::thread write_thread([&]() {
-        Result<std::unique_ptr<RecordBatch>> batch =
-            MakeBatch({Row{2, "two", "p0"}}, /*partitioned=*/false);
-        if (!batch.ok()) {
-            write_promise.set_value(batch.status());
-            return;
-        }
-        write_promise.set_value(writer->Write(std::move(batch).value()));
-    });
-    const bool write_completed =
-        write_future.wait_for(std::chrono::seconds(5)) == std::future_status::ready;
-    state->Release();
-    prepare_thread.join();
-    write_thread.join();
-
-    ASSERT_TRUE(write_completed);
-    ASSERT_OK(write_future.get());
-    ASSERT_OK(prepare_result);
-    ASSERT_EQ(1, prepare_result.value().size());
-    ASSERT_EQ(OffsetRange(0, 1), prepare_result.value()[0].offset_range);
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_progress,
-                         writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
-    ASSERT_EQ(1, second_progress.size());
-    ASSERT_EQ(OffsetRange(1, 2), second_progress[0].offset_range);
-    ASSERT_OK(writer->Close());
-}
-
 TEST_F(RealtimeWriteInteTest, TestPkPluginContract) {
     CreatePkTable();
     auto factory = std::make_shared<SplitCommitReaderRealtimeStoreFactory>();

From b6bf461806cb09f495bb6d62eb14b833a7453457 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Mon, 24 Aug 2026 14:49:07 +0800
Subject: [PATCH 17/62] test(mergetree): reuse reader failure mock

---
 .../core/mergetree/merge_tree_writer_test.cpp | 44 ++++++-------------
 1 file changed, 14 insertions(+), 30 deletions(-)

diff --git a/src/paimon/core/mergetree/merge_tree_writer_test.cpp b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
index 675ce3198..63e896573 100644
--- a/src/paimon/core/mergetree/merge_tree_writer_test.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
@@ -96,31 +96,7 @@ class TrackingKeyValueRecordReader : public KeyValueRecordReader {
     bool* closed_flag_;
 };
 
-class ErrorKeyValueRecordReader : public KeyValueRecordReader {
- public:
-    ErrorKeyValueRecordReader(Status status, bool* closed_flag)
-        : status_(std::move(status)), closed_flag_(closed_flag) {}
-
-    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatch() override {
-        return status_;
-    }
-
-    std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return nullptr;
-    }
-
-    void Close() override {
-        if (closed_flag_ != nullptr) {
-            *closed_flag_ = true;
-        }
-    }
-
- private:
-    Status status_;
-    bool* closed_flag_;
-};
-
-}
+}  // namespace
 
 class MergeTreeWriterTest : public ::testing::TestWithParam<bool> {
  public:
@@ -270,7 +246,8 @@ class MergeTreeWriterTest : public ::testing::TestWithParam<bool> {
     }
 
     std::unique_ptr<KeyValueRecordReader> CreateSingleReader(
-        const std::shared_ptr<arrow::Array>& array, int32_t batch_size = 16) const {
+        const std::shared_ptr<arrow::Array>& array, int32_t batch_size = 16,
+        const Status& next_batch_status = Status::OK()) const {
         std::vector<DataField> write_fields = {SpecialFields::SequenceNumber(),
                                                SpecialFields::ValueKind()};
         write_fields.insert(write_fields.end(), value_fields_.begin(), value_fields_.end());
@@ -280,6 +257,7 @@ class MergeTreeWriterTest : public ::testing::TestWithParam<bool> {
             arrow::schema(arrow::FieldVector({write_schema->field(2)}));
         auto file_batch_reader =
             std::make_unique<MockFileBatchReader>(array, array->type(), batch_size);
+        file_batch_reader->SetNextBatchStatus(next_batch_status);
         return std::make_unique<MockKeyValueDataFileRecordReader>(
             std::move(file_batch_reader), key_schema, value_schema_, 0, pool_);
     }
@@ -601,13 +579,19 @@ TEST_P(MergeTreeWriterTest, TestSortedReaderFailure) {
     Status null_status = merge_writer->WriteSortedReaders(std::move(null_readers));
     ASSERT_TRUE(null_status.IsInvalid());
 
+    auto sorted_reader_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(write_type_, R"([
+      [0, 0, "Alice", 10, 0, 13.1]
+    ])")
+            .ValueOrDie());
+    Status expected_status = Status::IOError("sorted reader failure");
     bool failing_reader_closed = false;
-    auto failing_reader = std::make_unique<ErrorKeyValueRecordReader>(
-        Status::IOError("sorted reader failure"), &failing_reader_closed);
     std::vector<std::unique_ptr<KeyValueRecordReader>> failing_readers;
-    failing_readers.push_back(std::move(failing_reader));
+    failing_readers.push_back(std::make_unique<TrackingKeyValueRecordReader>(
+        CreateSingleReader(sorted_reader_array, /*batch_size=*/16, expected_status),
+        &failing_reader_closed));
     Status failing_status = merge_writer->WriteSortedReaders(std::move(failing_readers));
-    ASSERT_TRUE(failing_status.IsIOError());
+    ASSERT_EQ(expected_status, failing_status);
     ASSERT_TRUE(failing_reader_closed);
     ASSERT_OK(merge_writer->Close());
 }

From 8c4c4c8b20754882eb2b2bc0749dcc9464b62aca Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Mon, 24 Aug 2026 14:49:38 +0800
Subject: [PATCH 18/62] fix(realtime): preserve PK sequence across writer
 handoff

---
 .../operation/key_value_file_store_write.cpp  |  6 +--
 .../core/realtime/realtime_context_impl.cpp   | 11 +++++
 .../core/realtime/realtime_context_impl.h     |  4 ++
 .../core/realtime/realtime_context_test.cpp   | 16 +++++++
 .../realtime/realtime_primary_key_writer.cpp  | 23 +++++++---
 .../realtime/realtime_primary_key_writer.h    | 10 ++++-
 test/inte/realtime_write_inte_test.cpp        | 45 +++++++++++++++++++
 7 files changed, 106 insertions(+), 9 deletions(-)

diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp
index d2c97abcf..de7217ec3 100644
--- a/src/paimon/core/operation/key_value_file_store_write.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write.cpp
@@ -175,9 +175,9 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
     if (!realtime_context_) {
         return std::shared_ptr<BatchWriter>(std::move(writer));
     }
-    return RealtimePrimaryKeyWriter::Create(schema_, trimmed_primary_keys, key_comparator_,
-                                            realtime_store_state.value(), restore_max_seq_number,
-                                            writer, pool_);
+    return RealtimePrimaryKeyWriter::Create(
+        partition_map, bucket, schema_, trimmed_primary_keys, key_comparator_,
+        realtime_context_impl, realtime_store_state.value(), restore_max_seq_number, writer, pool_);
 }
 
 Status KeyValueFileStoreWrite::RefreshCommittedSnapshot(int64_t snapshot_id) {
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index b73cfdb8a..415052a69 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -130,6 +130,17 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     return RealtimeStoreState{std::move(store), initial_offset};
 }
 
+int64_t RealtimeContextImpl::AdvanceMaterializedMaxSequenceNumber(
+    const RealtimePartitionBucket& partition_bucket, int64_t max_sequence_number) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    auto [iter, inserted] =
+        materialized_max_sequence_numbers_.emplace(partition_bucket, max_sequence_number);
+    if (!inserted && max_sequence_number > iter->second) {
+        iter->second = max_sequence_number;
+    }
+    return iter->second;
+}
+
 Result<std::vector<RealtimePartitionBucketView>> RealtimeContextImpl::AcquireReadViews() {
     std::lock_guard<std::mutex> lock(mutex_);
     std::vector<RealtimePartitionBucketView> result;
diff --git a/src/paimon/core/realtime/realtime_context_impl.h b/src/paimon/core/realtime/realtime_context_impl.h
index 4f62cf1ee..aa4d263c6 100644
--- a/src/paimon/core/realtime/realtime_context_impl.h
+++ b/src/paimon/core/realtime/realtime_context_impl.h
@@ -67,6 +67,9 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
 
     Result<RealtimeStoreState> GetOrCreateRealtimeStore(RealtimeStoreCreateRequest&& request);
 
+    int64_t AdvanceMaterializedMaxSequenceNumber(const RealtimePartitionBucket& partition_bucket,
+                                                 int64_t max_sequence_number);
+
     Result<std::vector<RealtimePartitionBucketView>> AcquireReadViews();
 
     Result<std::string> PinReadView(const RealtimePartitionBucketView& view, int64_t ttl_millis);
@@ -96,6 +99,7 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
     std::mutex mutex_;
     std::mutex progress_mutex_;
     std::map<RealtimePartitionBucket, std::shared_ptr<RealtimeStore>> stores_;
+    std::map<RealtimePartitionBucket, int64_t> materialized_max_sequence_numbers_;
     // Full-table progress used as the initial offset when a store is created lazily.
     RealtimeOffsetMap committed_offsets_;
     // Progress already reflected in stores owned by this context.
diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp
index 07bbf555b..15066ca1d 100644
--- a/src/paimon/core/realtime/realtime_context_test.cpp
+++ b/src/paimon/core/realtime/realtime_context_test.cpp
@@ -158,6 +158,22 @@ TEST(RealtimeContextTest, TestReusesStoreAndCapturesRegisteredViews) {
     ASSERT_EQ(1, factory->stores[2]->acquire_count);
 }
 
+TEST(RealtimeContextTest, TestReconcilesPrimaryKeyInitialSequence) {
+    auto factory = std::make_shared<TestingRealtimeStoreFactory>();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
+    const std::map<std::string, std::string> partition = {{"dt", "2026-08-02"}};
+    const RealtimePartitionBucket partition_bucket(partition, /*bucket=*/0);
+
+    ASSERT_EQ(4, context->AdvanceMaterializedMaxSequenceNumber(partition_bucket,
+                                                               /*max_sequence_number=*/4));
+    ASSERT_EQ(8, context->AdvanceMaterializedMaxSequenceNumber(partition_bucket,
+                                                               /*max_sequence_number=*/8));
+    ASSERT_EQ(8, context->AdvanceMaterializedMaxSequenceNumber(partition_bucket,
+                                                               /*max_sequence_number=*/6));
+    ASSERT_EQ(10, context->AdvanceMaterializedMaxSequenceNumber(partition_bucket,
+                                                                /*max_sequence_number=*/10));
+}
+
 TEST(RealtimeContextTest, TestCommittedProgressIsMonotonicAndSelective) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index 2dc5a71b4..b53831f0a 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -138,13 +138,16 @@ Result<std::shared_ptr<arrow::StructArray>> PrepareBatch(
 }  // namespace
 
 Result<std::shared_ptr<RealtimePrimaryKeyWriter>> RealtimePrimaryKeyWriter::Create(
+    const std::map<std::string, std::string>& partition, int32_t bucket,
     const std::shared_ptr<arrow::Schema>& write_schema,
     const std::vector<std::string>& trimmed_primary_keys,
-    const std::shared_ptr<FieldsComparator>& key_comparator, const RealtimeStoreState& store_state,
-    int64_t restored_max_sequence_number, const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
+    const std::shared_ptr<FieldsComparator>& key_comparator,
+    const std::shared_ptr<RealtimeContextImpl>& realtime_context,
+    const RealtimeStoreState& store_state, int64_t restored_max_sequence_number,
+    const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
     const std::shared_ptr<MemoryPool>& memory_pool) {
     if (!store_state.store || !merge_tree_writer || !write_schema || !key_comparator ||
-        !memory_pool) {
+        !realtime_context || !memory_pool) {
         return Status::Invalid("PK real-time writer received a null dependency");
     }
     if (trimmed_primary_keys.empty()) {
@@ -170,16 +173,22 @@ Result<std::shared_ptr<RealtimePrimaryKeyWriter>> RealtimePrimaryKeyWriter::Crea
         DataField::ConvertDataFieldToArrowField(RealtimeOffsetField())->WithNullable(false)};
     prepared_fields.insert(prepared_fields.end(), write_schema->fields().begin(),
                            write_schema->fields().end());
+    const RealtimePartitionBucket partition_bucket(partition, bucket);
+    const int64_t initial_max_sequence_number =
+        realtime_context->AdvanceMaterializedMaxSequenceNumber(partition_bucket,
+                                                               restored_max_sequence_number);
     return std::shared_ptr<RealtimePrimaryKeyWriter>(new RealtimePrimaryKeyWriter(
-        store_state.store, merge_tree_writer, write_schema,
+        store_state.store, merge_tree_writer, realtime_context, partition_bucket, write_schema,
         arrow::schema(std::move(prepared_fields)), arrow::schema(std::move(key_fields)),
         trimmed_primary_keys, key_comparator, store_state.initial_offset,
-        restored_max_sequence_number, memory_pool));
+        initial_max_sequence_number, memory_pool));
 }
 
 RealtimePrimaryKeyWriter::RealtimePrimaryKeyWriter(
     const std::shared_ptr<RealtimeStore>& realtime_store,
     const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
+    const std::shared_ptr<RealtimeContextImpl>& realtime_context,
+    const RealtimePartitionBucket& partition_bucket,
     const std::shared_ptr<arrow::Schema>& write_schema,
     const std::shared_ptr<arrow::Schema>& prepared_schema,
     const std::shared_ptr<arrow::Schema>& key_schema,
@@ -190,6 +199,8 @@ RealtimePrimaryKeyWriter::RealtimePrimaryKeyWriter(
       arrow_pool_(GetArrowPool(memory_pool)),
       realtime_store_(realtime_store),
       merge_tree_writer_(merge_tree_writer),
+      realtime_context_(realtime_context),
+      partition_bucket_(partition_bucket),
       write_schema_(write_schema),
       prepared_schema_(prepared_schema),
       key_schema_(key_schema),
@@ -237,6 +248,8 @@ Status RealtimePrimaryKeyWriter::Write(std::unique_ptr<RecordBatch>&& batch) {
         std::move(prepared_batch), OffsetRange(next_offset_, next_offset_ + count)}));
     next_offset_ += count;
     last_sequence_number_ += count;
+    realtime_context_->AdvanceMaterializedMaxSequenceNumber(partition_bucket_,
+                                                            last_sequence_number_);
     return Status::OK();
 }
 
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.h b/src/paimon/core/realtime/realtime_primary_key_writer.h
index 6abb1ccd0..9a5aa4c68 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.h
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.h
@@ -26,6 +26,7 @@
 #include <vector>
 
 #include "paimon/core/utils/batch_writer.h"
+#include "paimon/realtime/realtime_context.h"
 #include "paimon/realtime/realtime_store.h"
 
 namespace arrow {
@@ -38,16 +39,19 @@ namespace paimon {
 class MemoryPool;
 class MergeTreeWriter;
 class FieldsComparator;
+class RealtimeContextImpl;
 struct RealtimeStoreState;
 
 /// Coordinates framework-prepared primary-key real-time writes.
 class RealtimePrimaryKeyWriter final : public BatchWriter {
  public:
     static Result<std::shared_ptr<RealtimePrimaryKeyWriter>> Create(
+        const std::map<std::string, std::string>& partition, int32_t bucket,
         const std::shared_ptr<arrow::Schema>& write_schema,
         const std::vector<std::string>& trimmed_primary_keys,
         const std::shared_ptr<FieldsComparator>& key_comparator,
-        const RealtimeStoreState& store_state, int64_t restore_max_sequence_number,
+        const std::shared_ptr<RealtimeContextImpl>& realtime_context,
+        const RealtimeStoreState& store_state, int64_t restored_max_sequence_number,
         const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
         const std::shared_ptr<MemoryPool>& memory_pool);
 
@@ -64,6 +68,8 @@ class RealtimePrimaryKeyWriter final : public BatchWriter {
  private:
     RealtimePrimaryKeyWriter(const std::shared_ptr<RealtimeStore>& realtime_store,
                              const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
+                             const std::shared_ptr<RealtimeContextImpl>& realtime_context,
+                             const RealtimePartitionBucket& partition_bucket,
                              const std::shared_ptr<arrow::Schema>& write_schema,
                              const std::shared_ptr<arrow::Schema>& prepared_schema,
                              const std::shared_ptr<arrow::Schema>& key_schema,
@@ -79,6 +85,8 @@ class RealtimePrimaryKeyWriter final : public BatchWriter {
     std::shared_ptr<arrow::MemoryPool> arrow_pool_;
     std::shared_ptr<RealtimeStore> realtime_store_;
     std::shared_ptr<MergeTreeWriter> merge_tree_writer_;
+    std::shared_ptr<RealtimeContextImpl> realtime_context_;
+    RealtimePartitionBucket partition_bucket_;
     std::shared_ptr<arrow::Schema> write_schema_;
     std::shared_ptr<arrow::Schema> prepared_schema_;
     std::shared_ptr<arrow::Schema> key_schema_;
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index c61b04b0d..e68ff670d 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -1526,6 +1526,51 @@ TEST_F(RealtimeWriteInteTest, TestPkCompositeMerge) {
     ASSERT_OK(writer->Close());
 }
 
+TEST_F(RealtimeWriteInteTest, TestPkWriterHandoff) {
+    CreatePkTable();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> first_writer,
+                         CreateRealtimeWriter(realtime_context));
+    const std::vector<Row> first_rows = {
+        {0, "value-0", "p0"}, {1, "value-1", "p0"}, {2, "value-2", "p0"}};
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
+                         MakeBatch(first_rows, /*partitioned=*/false));
+    ASSERT_OK(first_writer->Write(std::move(first_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> first_progress,
+                         first_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_EQ(1, first_progress.size());
+    ASSERT_EQ(OffsetRange(0, 3), first_progress[0].offset_range);
+    ASSERT_EQ(1, NewFiles(first_progress).size());
+    ASSERT_EQ(0, NewFiles(first_progress)[0]->min_sequence_number);
+    ASSERT_EQ(2, NewFiles(first_progress)[0]->max_sequence_number);
+    ASSERT_OK(first_writer->Close());
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> second_writer,
+                         CreateRealtimeWriter(realtime_context));
+    const std::vector<Row> second_rows = {{0, "updated-0", "p0"}, {3, "value-3", "p0"}};
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
+                         MakeBatch(second_rows, /*partitioned=*/false));
+    ASSERT_OK(second_writer->Write(std::move(second_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_progress,
+                         second_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_EQ(1, second_progress.size());
+    ASSERT_EQ(OffsetRange(3, 5), second_progress[0].offset_range);
+    ASSERT_EQ(1, NewFiles(second_progress).size());
+    ASSERT_EQ(3, NewFiles(second_progress)[0]->min_sequence_number);
+    ASSERT_EQ(4, NewFiles(second_progress)[0]->max_sequence_number);
+
+    first_progress.push_back(std::move(second_progress[0]));
+    ASSERT_OK(Commit(first_progress, /*commit_identifier=*/1));
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(realtime_context));
+    ASSERT_EQ((std::vector<Row>{{0, "updated-0", "p0"},
+                                {1, "value-1", "p0"},
+                                {2, "value-2", "p0"},
+                                {3, "value-3", "p0"}}),
+              actual_rows);
+    ASSERT_OK(second_writer->Close());
+}
+
 TEST_F(RealtimeWriteInteTest, TestPkPartitionBucketRecovery) {
     options_[Options::BUCKET] = "2";
     CreatePkTable(/*partition_keys=*/{"pt"});

From aec0a2ec2dc3015cfc5f6168a22f1adcd060c2c5 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Mon, 24 Aug 2026 15:56:00 +0800
Subject: [PATCH 19/62] refactor(realtime): simplify primary key merge readers

---
 .../core/operation/merge_file_split_read.cpp  | 196 +++---------------
 .../core/operation/merge_file_split_read.h    |   9 +-
 .../table/source/key_value_table_read.cpp     |  21 +-
 3 files changed, 35 insertions(+), 191 deletions(-)

diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp
index 2f64f6df8..c85e75ee0 100644
--- a/src/paimon/core/operation/merge_file_split_read.cpp
+++ b/src/paimon/core/operation/merge_file_split_read.cpp
@@ -30,7 +30,6 @@
 #include "arrow/c/bridge.h"
 #include "arrow/type.h"
 #include "fmt/format.h"
-#include "paimon/common/metrics/metrics_impl.h"
 #include "paimon/common/reader/complete_row_kind_batch_reader.h"
 #include "paimon/common/reader/concat_batch_reader.h"
 #include "paimon/common/table/special_fields.h"
@@ -79,82 +78,36 @@ struct KeyValue;
 template <typename T>
 class MergeFunctionWrapper;
 
-namespace {
-
-class ConcatNonOverlappingMergeReaders final : public SortMergeReader {
- public:
-    explicit ConcatNonOverlappingMergeReaders(
-        std::vector<std::unique_ptr<SortMergeReader>>&& readers)
-        : readers_(std::move(readers)) {}
-
-    Result<std::unique_ptr<SortMergeReader::Iterator>> NextBatch() override {
-        while (current_ < readers_.size()) {
-            PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<SortMergeReader::Iterator> iterator,
-                                   readers_[current_]->NextBatch());
-            if (iterator) {
-                return iterator;
-            }
-            readers_[current_]->Close();
-            ++current_;
-        }
-        return std::unique_ptr<SortMergeReader::Iterator>();
-    }
-
-    void Close() override {
-        while (current_ < readers_.size()) {
-            readers_[current_++]->Close();
-        }
-    }
-
-    std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return MetricsImpl::CollectReadMetrics(readers_);
-    }
-
- private:
-    std::vector<std::unique_ptr<SortMergeReader>> readers_;
-    size_t current_ = 0;
-};
-
-}
-
 class MergeFileSplitRead::RealtimeReaderBuilder {
  public:
     static Result<std::unique_ptr<BatchReader>> Create(
         MergeFileSplitRead* owner, const std::vector<std::shared_ptr<Split>>& disk_splits,
-        std::vector<AdditionalKeyValueReader>&& additional_readers) {
+        std::vector<std::unique_ptr<KeyValueRecordReader>>&& additional_readers) {
         RealtimeReaderBuilder builder(owner);
-        if (disk_splits.empty()) {
-            std::vector<std::unique_ptr<KeyValueRecordReader>> readers;
-            readers.reserve(additional_readers.size());
-            for (AdditionalKeyValueReader& additional : additional_readers) {
-                readers.push_back(std::move(additional.reader));
-            }
-            return builder.CreateMergedReader(std::move(readers));
+        std::vector<std::unique_ptr<KeyValueRecordReader>> readers;
+        if (!disk_splits.empty()) {
+            PAIMON_RETURN_NOT_OK(builder.CollectDiskReaders(disk_splits, &readers));
         }
-
-        PAIMON_RETURN_NOT_OK(builder.CollectDiskInputs(disk_splits));
-        builder.AddRangeInputs(std::move(additional_readers));
-        return builder.CreateReader();
+        readers.reserve(readers.size() + additional_readers.size());
+        for (std::unique_ptr<KeyValueRecordReader>& additional_reader : additional_readers) {
+            readers.push_back(std::move(additional_reader));
+        }
+        return builder.CreateMergedReader(std::move(readers));
     }
 
  private:
-    struct RangeInput {
-        std::shared_ptr<InternalRow> min_key;
-        std::shared_ptr<InternalRow> max_key;
-        std::vector<SortedRun> disk_runs;
-        std::unique_ptr<KeyValueRecordReader> additional_reader;
-    };
-
     explicit RealtimeReaderBuilder(MergeFileSplitRead* owner) : owner_(owner) {}
 
-    Status CollectDiskInputs(const std::vector<std::shared_ptr<Split>>& disk_splits) {
-        first_split_ = std::dynamic_pointer_cast<DataSplitImpl>(disk_splits.front());
-        if (!first_split_) {
+    Status CollectDiskReaders(const std::vector<std::shared_ptr<Split>>& disk_splits,
+                              std::vector<std::unique_ptr<KeyValueRecordReader>>* readers) {
+        std::shared_ptr<DataSplitImpl> first_split =
+            std::dynamic_pointer_cast<DataSplitImpl>(disk_splits.front());
+        if (!first_split) {
             return Status::Invalid("merge input disk split is not a data split");
         }
-        const BinaryRow& partition = first_split_->Partition();
-        const int32_t bucket = first_split_->Bucket();
-        PAIMON_ASSIGN_OR_RAISE(data_file_path_factory_,
+        const BinaryRow& partition = first_split->Partition();
+        const int32_t bucket = first_split->Bucket();
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<DataFilePathFactory> data_file_path_factory,
                                owner_->path_factory_->CreateDataFilePathFactory(partition, bucket));
 
         std::vector<std::shared_ptr<DataFileMeta>> data_files;
@@ -187,46 +140,23 @@ class MergeFileSplitRead::RealtimeReaderBuilder {
             }
         }
 
-        dv_factory_ = DeletionVector::CreateFactory(
+        DeletionVector::Factory dv_factory = DeletionVector::CreateFactory(
             owner_->options_.GetFileSystem(),
             DeletionVector::CreateDeletionFileMap(data_files, deletion_files), owner_->pool_);
         std::vector<std::vector<SortedRun>> disk_sections =
             IntervalPartition(data_files, owner_->key_comparator_).Partition();
-        inputs_.reserve(disk_sections.size());
-        for (std::vector<SortedRun>& section : disk_sections) {
-            std::shared_ptr<DataFileMeta> min_file = section.front().Files().front();
-            std::shared_ptr<DataFileMeta> max_file = min_file;
+        for (const std::vector<SortedRun>& section : disk_sections) {
             for (const SortedRun& run : section) {
-                for (const std::shared_ptr<DataFileMeta>& file : run.Files()) {
-                    if (owner_->key_comparator_->CompareTo(file->min_key, min_file->min_key) < 0) {
-                        min_file = file;
-                    }
-                    if (owner_->key_comparator_->CompareTo(file->max_key, max_file->max_key) > 0) {
-                        max_file = file;
-                    }
-                }
+                PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> disk_reader,
+                                       owner_->CreateReaderForRun(partition, run, dv_factory,
+                                                                  owner_->predicate_for_keys_,
+                                                                  data_file_path_factory));
+                readers->push_back(std::move(disk_reader));
             }
-            inputs_.push_back(RangeInput{std::shared_ptr<InternalRow>(min_file, &min_file->min_key),
-                                         std::shared_ptr<InternalRow>(max_file, &max_file->max_key),
-                                         std::move(section), nullptr});
         }
         return Status::OK();
     }
 
-    void AddRangeInputs(std::vector<AdditionalKeyValueReader>&& additional_readers) {
-        inputs_.reserve(inputs_.size() + additional_readers.size());
-        for (AdditionalKeyValueReader& additional : additional_readers) {
-            has_unknown_range_ |= !additional.min_key || !additional.max_key;
-            inputs_.push_back(RangeInput{additional.min_key, additional.max_key, {},
-                                         std::move(additional.reader)});
-        }
-    }
-
-    Result<std::unique_ptr<KeyValueRecordReader>> CreateDiskReader(const SortedRun& run) {
-        return owner_->CreateReaderForRun(first_split_->Partition(), run, dv_factory_,
-                                          owner_->predicate_for_keys_, data_file_path_factory_);
-    }
-
     Result<std::unique_ptr<BatchReader>> CreateMergedReader(
         std::vector<std::unique_ptr<KeyValueRecordReader>>&& record_readers) {
         if (record_readers.empty()) {
@@ -265,83 +195,7 @@ class MergeFileSplitRead::RealtimeReaderBuilder {
                                                             owner_->pool_);
     }
 
-    Result<std::unique_ptr<BatchReader>> CreateUnknownRangeReader() {
-        std::vector<std::unique_ptr<KeyValueRecordReader>> readers;
-        for (RangeInput& input : inputs_) {
-            for (const SortedRun& run : input.disk_runs) {
-                PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> disk_reader,
-                                       CreateDiskReader(run));
-                readers.push_back(std::move(disk_reader));
-            }
-            if (input.additional_reader) {
-                readers.push_back(std::move(input.additional_reader));
-            }
-        }
-        return CreateMergedReader(std::move(readers));
-    }
-
-    Result<std::unique_ptr<BatchReader>> CreateKnownRangeReader() {
-        std::sort(inputs_.begin(), inputs_.end(),
-                  [this](const RangeInput& lhs, const RangeInput& rhs) {
-                      return owner_->key_comparator_->CompareTo(*lhs.min_key, *rhs.min_key) < 0;
-                  });
-        std::vector<std::vector<RangeInput>> components;
-        std::shared_ptr<InternalRow> component_max_key;
-        for (RangeInput& input : inputs_) {
-            if (components.empty() ||
-                owner_->key_comparator_->CompareTo(*input.min_key, *component_max_key) > 0) {
-                components.emplace_back();
-                component_max_key = input.max_key;
-            } else if (owner_->key_comparator_->CompareTo(*input.max_key, *component_max_key) > 0) {
-                component_max_key = input.max_key;
-            }
-            components.back().push_back(std::move(input));
-        }
-
-        std::vector<std::unique_ptr<SortMergeReader>> component_readers;
-        component_readers.reserve(components.size());
-        for (std::vector<RangeInput>& component : components) {
-            if (component.size() == 1 && !component.front().additional_reader) {
-                PAIMON_ASSIGN_OR_RAISE(
-                    std::unique_ptr<SortMergeReader> disk_component,
-                    owner_->CreateSortMergeReaderForSection(
-                        component.front().disk_runs, first_split_->Partition(), dv_factory_,
-                        component.front().disk_runs.size() == 1 ? owner_->context_->GetPredicate()
-                                                                : owner_->predicate_for_keys_,
-                        data_file_path_factory_, false));
-                component_readers.push_back(std::move(disk_component));
-                continue;
-            }
-
-            std::vector<std::unique_ptr<KeyValueRecordReader>> readers;
-            for (RangeInput& input : component) {
-                for (const SortedRun& run : input.disk_runs) {
-                    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> disk_reader,
-                                           CreateDiskReader(run));
-                    readers.push_back(std::move(disk_reader));
-                }
-                if (input.additional_reader) {
-                    readers.push_back(std::move(input.additional_reader));
-                }
-            }
-            PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<SortMergeReader> component_reader,
-                                   owner_->CreateSortMergeReader(std::move(readers)));
-            component_readers.push_back(std::move(component_reader));
-        }
-        return CreateProjectedReader(
-            std::make_unique<ConcatNonOverlappingMergeReaders>(std::move(component_readers)));
-    }
-
-    Result<std::unique_ptr<BatchReader>> CreateReader() {
-        return has_unknown_range_ ? CreateUnknownRangeReader() : CreateKnownRangeReader();
-    }
-
     MergeFileSplitRead* owner_;
-    std::shared_ptr<DataSplitImpl> first_split_;
-    std::shared_ptr<DataFilePathFactory> data_file_path_factory_;
-    DeletionVector::Factory dv_factory_;
-    std::vector<RangeInput> inputs_;
-    bool has_unknown_range_ = false;
 };
 
 Result<std::unique_ptr<MergeFileSplitRead>> MergeFileSplitRead::Create(
@@ -426,7 +280,7 @@ Result<std::unique_ptr<BatchReader>> MergeFileSplitRead::CreateReader(
 
 Result<std::unique_ptr<BatchReader>> MergeFileSplitRead::CreateRealtimeReader(
     const std::vector<std::shared_ptr<Split>>& disk_splits,
-    std::vector<AdditionalKeyValueReader>&& additional_readers) {
+    std::vector<std::unique_ptr<KeyValueRecordReader>>&& additional_readers) {
     return RealtimeReaderBuilder::Create(this, disk_splits, std::move(additional_readers));
 }
 
diff --git a/src/paimon/core/operation/merge_file_split_read.h b/src/paimon/core/operation/merge_file_split_read.h
index 85b5b2a28..f01b252be 100644
--- a/src/paimon/core/operation/merge_file_split_read.h
+++ b/src/paimon/core/operation/merge_file_split_read.h
@@ -55,7 +55,6 @@ class FieldsComparator;
 class FileBatchReader;
 class FileStorePathFactory;
 class InternalReadContext;
-class InternalRow;
 class MemoryPool;
 class SchemaManager;
 class SortedRun;
@@ -66,12 +65,6 @@ struct KeyValue;
 template <typename T>
 class MergeFunctionWrapper;
 
-struct AdditionalKeyValueReader {
-    std::unique_ptr<KeyValueRecordReader> reader;
-    std::shared_ptr<InternalRow> min_key;
-    std::shared_ptr<InternalRow> max_key;
-};
-
 /// If the class name below is enclosed in parentheses, it might be present in the read path;
 /// otherwise, it must be present in the read path.
 ///
@@ -129,7 +122,7 @@ class MergeFileSplitRead : public AbstractSplitRead {
 
     Result<std::unique_ptr<BatchReader>> CreateRealtimeReader(
         const std::vector<std::shared_ptr<Split>>& disk_splits,
-        std::vector<AdditionalKeyValueReader>&& additional_readers);
+        std::vector<std::unique_ptr<KeyValueRecordReader>>&& additional_readers);
 
     void SetMergeFunctionWrapper(
         const std::shared_ptr<MergeFunctionWrapper<KeyValue>>& merge_function_wrapper);
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index 31779e049..dc69ebb55 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -55,7 +55,7 @@ struct ColumnarBatchContext;
 
 namespace {
 
-Result<std::vector<AdditionalKeyValueReader>> CreateMemoryReaders(
+Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateMemoryReaders(
     const std::shared_ptr<RealtimeSplit>& split, const RealtimePartitionBucketView& memory,
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
@@ -76,9 +76,8 @@ Result<std::vector<AdditionalKeyValueReader>> CreateMemoryReaders(
     PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*prepared_schema, c_schema.get()));
     ScopeGuard schema_guard([schema = c_schema.get()]() { ArrowSchemaRelease(schema); });
     RealtimeQueryContext query_context{c_schema.get(), nullptr, false};
-    PAIMON_ASSIGN_OR_RAISE(
-        std::vector<std::unique_ptr<BatchReader>> batch_readers,
-        memory.store->CreateQueryReaders(memory.read_view, 0, query_context));
+    PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> batch_readers,
+                           memory.store->CreateQueryReaders(memory.read_view, 0, query_context));
     ScopeGuard batch_readers_guard([&batch_readers]() {
         for (const std::unique_ptr<BatchReader>& reader : batch_readers) {
             if (reader) {
@@ -86,7 +85,7 @@ Result<std::vector<AdditionalKeyValueReader>> CreateMemoryReaders(
             }
         }
     });
-    std::vector<AdditionalKeyValueReader> result;
+    std::vector<std::unique_ptr<KeyValueRecordReader>> result;
     result.reserve(batch_readers.size());
     for (std::unique_ptr<BatchReader>& reader : batch_readers) {
         if (!reader) {
@@ -98,17 +97,15 @@ Result<std::vector<AdditionalKeyValueReader>> CreateMemoryReaders(
                                                                     split->MemoryEndOffset()),
                                                         key_schema, value_schema, memory_pool));
         auto merge = std::make_unique<DeduplicateMergeFunction>(false);
-        result.push_back(AdditionalKeyValueReader{
-            std::make_unique<MergedKeyValueRecordReader>(
-                std::move(prepared_reader), key_comparator,
-                std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge))),
-            nullptr, nullptr});
+        result.push_back(std::make_unique<MergedKeyValueRecordReader>(
+            std::move(prepared_reader), key_comparator,
+            std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge))));
     }
     batch_readers_guard.Release();
     return result;
 }
 
-}
+}  // namespace
 
 KeyValueTableRead::KeyValueTableRead(std::vector<std::unique_ptr<SplitRead>>&& split_reads,
                                      const std::shared_ptr<FileStorePathFactory>& path_factory,
@@ -268,7 +265,7 @@ Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateRealtimeReader(
         auto* merge_read = dynamic_cast<MergeFileSplitRead*>(read.get());
         if (merge_read) {
             PAIMON_ASSIGN_OR_RAISE(
-                std::vector<AdditionalKeyValueReader> memory_readers,
+                std::vector<std::unique_ptr<KeyValueRecordReader>> memory_readers,
                 CreateMemoryReaders(realtime_split, memory, merge_read->GetKeySchema(),
                                     merge_read->GetValueSchema(), merge_read->GetKeyComparator(),
                                     context_, GetMemoryPool()));

From c248373d918afce1f9df9b8338bb69d9789ba17c Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Mon, 24 Aug 2026 17:20:13 +0800
Subject: [PATCH 20/62] fix(realtime): validate PK reader contracts

---
 include/paimon/realtime/realtime_store.h      |  11 +-
 .../merged_key_value_record_reader_test.cpp   |  98 ++----
 .../core/operation/file_store_write.cpp       |   2 +-
 .../operation/key_value_file_store_write.cpp  |   2 +-
 .../realtime/arrow_realtime_store_factory.cpp |   5 +-
 .../realtime/prepared_key_value_reader.cpp    | 231 ++++++++++++--
 .../core/realtime/prepared_key_value_reader.h |  22 +-
 .../realtime/primary_key_realtime_store.cpp   | 160 ++++++++--
 .../realtime/primary_key_realtime_store.h     |   4 +-
 .../primary_key_realtime_store_test.cpp       | 144 ++++++++-
 .../core/realtime/realtime_context_impl.cpp   |  55 +++-
 .../core/realtime/realtime_context_impl.h     |  12 +-
 .../core/realtime/realtime_context_test.cpp   |  27 +-
 .../realtime/realtime_primary_key_writer.cpp  |  32 +-
 .../realtime/realtime_primary_key_writer.h    |   2 +-
 src/paimon/core/realtime/realtime_reader.h    |  11 +
 .../core/realtime/realtime_reader_test.cpp    |  27 +-
 .../table/source/key_value_table_read.cpp     |  11 +-
 src/paimon/core/table/source/table_scan.cpp   |   2 +-
 test/inte/realtime_write_inte_test.cpp        | 300 +++++++++++++++++-
 20 files changed, 971 insertions(+), 187 deletions(-)

diff --git a/include/paimon/realtime/realtime_store.h b/include/paimon/realtime/realtime_store.h
index dc5d543ac..792bb1c56 100644
--- a/include/paimon/realtime/realtime_store.h
+++ b/include/paimon/realtime/realtime_store.h
@@ -47,7 +47,10 @@ struct PAIMON_EXPORT AppendRealtimeStoreCreateConfig {
     StatisticsMode statistics_mode;
 };
 
-struct PAIMON_EXPORT PrimaryKeyRealtimeStoreCreateConfig {};
+struct PAIMON_EXPORT PrimaryKeyRealtimeStoreCreateConfig {
+    /// Primary-key fields after removing partition fields, in comparison order.
+    std::vector<std::string> trimmed_primary_keys;
+};
 
 using RealtimeStoreCreateConfig =
     std::variant<AppendRealtimeStoreCreateConfig, PrimaryKeyRealtimeStoreCreateConfig>;
@@ -148,7 +151,8 @@ class PAIMON_EXPORT RealtimeStore {
     /// including across `NextBatch` boundaries, is sorted by full primary key then sequence
     /// number; all readers collectively cover sealed mutations exactly once. Reader cardinality is
     /// independent of the number of writes. Paimon adapts and merges those rows before writing
-    /// files.
+    /// files. Paimon validates the complete ordering and coverage before publishing generated file
+    /// state; a violation fails the prepare operation.
     virtual Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
         const std::shared_ptr<RealtimeSegmentHandle>& segment) = 0;
 
@@ -168,7 +172,8 @@ class PAIMON_EXPORT RealtimeStore {
     /// contain multiple mutations per key. Each returned primary-key reader's complete stream is
     /// sorted by full primary key then sequence number, and all readers collectively cover raw
     /// mutations exactly once. Reader cardinality is independent of the number of writes. Paimon
-    /// retains `view` for the lifetime of the resulting framework reader.
+    /// validates ordering while adapting each complete reader stream and retains `view` for the
+    /// lifetime of the resulting framework reader.
     virtual Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
         const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
         const RealtimeQueryContext& context) = 0;
diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index 21b0a16b1..775f271e3 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -95,7 +95,7 @@ class TrackingBatchReader : public BatchReader {
     int32_t* close_count_;
 };
 
-}
+}  // namespace
 
 class MergedKeyValueRecordReaderTest : public testing::Test {
  public:
@@ -229,8 +229,7 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderOffsetFilter) {
     ])")
             .ValueOrDie());
 
-    auto batch_reader =
-        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 2);
+    auto batch_reader = std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 2);
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> reader,
         AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, OffsetRange(2, 4),
@@ -248,77 +247,28 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderOffsetFilter) {
     KeyValueChecker::CheckResult(expected, results, 1, 2);
 }
 
-TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderRawCountBeforeDedup) {
-    std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32())),
-                                           DataField(1, arrow::field("v0", arrow::int32()))};
+TEST_F(MergedKeyValueRecordReaderTest, TestRejectsUnsortedPluginRowsAcrossBatches) {
+    std::vector<DataField> value_fields = {DataField(0, arrow::field("id", arrow::int32()))};
     std::shared_ptr<arrow::Schema> value_schema =
         DataField::ConvertDataFieldsToArrowSchema(value_fields);
     std::shared_ptr<arrow::Schema> key_schema = arrow::schema({value_schema->field(0)});
     std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema(value_schema->fields());
     std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
-        [0, 10, 0, 1, 100],
-        [2, 11, 1, 1, 101],
-        [0, 12, 2, 2, 200]
-    ])")
-            .ValueOrDie());
-
-    int64_t raw_row_count = 0;
-    auto batch_reader =
-        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 2);
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> raw_reader,
-        AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, std::nullopt, key_schema,
-                                 value_schema, pool_, &raw_row_count));
-
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<FieldsComparator> key_comparator,
-                         FieldsComparator::Create({value_fields[0]}, true));
-    auto merged_reader = std::make_unique<MergedKeyValueRecordReader>(
-        std::move(raw_reader), key_comparator, merge_function_wrapper_);
-    ASSERT_OK_AND_ASSIGN(
-        std::vector<KeyValue> results,
-        (ReadResultCollector::CollectKeyValueResult<
-            MergedKeyValueRecordReader, KeyValueRecordReader::Iterator>(merged_reader.get())));
-
-    ASSERT_EQ(raw_row_count, 3);
-    std::vector<RowKind*> row_kinds = {const_cast<RowKind*>(RowKind::UpdateAfter()),
-                                       const_cast<RowKind*>(RowKind::Insert())};
-    std::vector<int64_t> levels = {KeyValue::UNKNOWN_LEVEL, KeyValue::UNKNOWN_LEVEL};
-    std::vector<KeyValue> expected = KeyValueChecker::GenerateKeyValues(
-        row_kinds, {11, 12}, levels, {{1}, {2}}, {{1, 101}, {2, 200}}, pool_);
-    KeyValueChecker::CheckResult(expected, results, 1, 2);
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderRawCountBeforeFilter) {
-    std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32()))};
-    std::shared_ptr<arrow::Schema> value_schema =
-        DataField::ConvertDataFieldsToArrowSchema(value_fields);
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema(value_schema->fields());
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
+    std::shared_ptr<arrow::Array> prepared_array =
         arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
-        [0, 10, 0, 1],
-        [0, 11, 1, 2],
-        [0, 12, 2, 3],
-        [0, 13, 3, 4]
-    ])")
-            .ValueOrDie());
-
-    int64_t raw_row_count = 0;
+            [0, 10, 0, 2],
+            [0, 11, 1, 1]
+        ])")
+            .ValueOrDie();
     auto batch_reader =
-        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 2);
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, OffsetRange(2, 4),
-                                 value_schema, value_schema, pool_, &raw_row_count));
-    ASSERT_OK_AND_ASSIGN(
-        std::vector<KeyValue> results,
-        (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
-                                                    KeyValueRecordReader::Iterator>(reader.get())));
-
-    ASSERT_EQ(results.size(), 2);
-    ASSERT_EQ(raw_row_count, 4);
+        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, /*batch_size=*/1);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
+                         AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema,
+                                                  std::nullopt, key_schema, value_schema, pool_));
+    Result<std::vector<KeyValue>> result =
+        ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
+                                                   KeyValueRecordReader::Iterator>(reader.get());
+    ASSERT_NOK_WITH_MSG(result, "not globally sorted by primary key and sequence number");
 }
 
 TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderCommitSchema) {
@@ -345,8 +295,7 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderCommitSchema) {
     ASSERT_EQ(query_results[0].value->GetFieldCount(), 1);
     ASSERT_EQ(query_results[0].value->GetInt(0), 1);
 
-    auto batch_reader =
-        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
+    auto batch_reader = std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
     ASSERT_NOK_WITH_MSG(AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema,
                                                  std::nullopt, value_schema, value_schema, pool_),
                         "exact");
@@ -364,8 +313,7 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderSafeDecode) {
     auto invalid_array = std::dynamic_pointer_cast<arrow::StructArray>(
         arrow::ipc::internal::json::ArrayFromJSON(invalid_type, R"([[0, 10, 0, 1]])").ValueOrDie());
 
-    auto batch_reader =
-        std::make_unique<MockFileBatchReader>(invalid_array, invalid_type, 1);
+    auto batch_reader = std::make_unique<MockFileBatchReader>(invalid_array, invalid_type, 1);
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> reader,
         AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
@@ -398,9 +346,12 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderNestedProjection) {
     std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
     auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
         arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
-        [0, 10, 0, 1, [[100, 200], [300, 400]], [["k1", [7, 8]], ["k2", [9, 10]]], [[[11, 12], 13], [[21, 22], 23]]]
+        [0, 9, 9, 0, [[1, 2]], [["prefix", [3, 4]]], [[[5, 6], 7]]],
+        [0, 10, 0, 1, [[100, 200], [300, 400]], [["k1", [7, 8]], ["k2", [9, 10]]], [[[11, 12], 13], [[21, 22], 23]]],
+        [0, 11, 11, 2, [[8, 9]], [["suffix", [10, 11]]], [[[12, 13], 14]]]
     ])")
             .ValueOrDie());
+    prepared_array = checked_pointer_cast<arrow::StructArray>(prepared_array->Slice(1, 1));
 
     std::shared_ptr<arrow::Field> query_item_b = MakeField("renamed_b", arrow::int32(), 11);
     std::shared_ptr<arrow::Field> query_item_a = MakeField("renamed_a", arrow::int32(), 10);
@@ -420,8 +371,7 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderNestedProjection) {
     std::shared_ptr<arrow::Schema> query_value_schema =
         arrow::schema({id, query_items, query_attrs, query_keyed_values});
 
-    auto batch_reader =
-        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
+    auto batch_reader = std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> reader,
         AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp
index 4d4f45156..84a324762 100644
--- a/src/paimon/core/operation/file_store_write.cpp
+++ b/src/paimon/core/operation/file_store_write.cpp
@@ -198,7 +198,7 @@ Result<std::unique_ptr<FileStoreWrite>> FileStoreWrite::Create(std::unique_ptr<W
     } else {
         // pk table
         if (ctx->GetRealtimeContext()) {
-            PAIMON_RETURN_NOT_OK(ValidatePrimaryKeyRealtimeOptions(options));
+            PAIMON_RETURN_NOT_OK(ValidatePrimaryKeyRealtimeOptions(options, *schema));
             if (ignore_previous_files) {
                 return Status::NotImplemented(
                     "PK realtime v1 requires restore from the latest snapshot");
diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp
index de7217ec3..ee6445057 100644
--- a/src/paimon/core/operation/key_value_file_store_write.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write.cpp
@@ -155,7 +155,7 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
             RealtimeStoreState store_state,
             realtime_context_impl->GetOrCreateRealtimeStore(RealtimeStoreCreateRequest{
                 std::move(c_write_schema), options_.ToMap(), pool_, partition_map, bucket,
-                PrimaryKeyRealtimeStoreCreateConfig{}}));
+                PrimaryKeyRealtimeStoreCreateConfig{trimmed_primary_keys}}));
         realtime_store_state = std::move(store_state);
         compact_manager = std::make_shared<NoopCompactManager>();
     } else {
diff --git a/src/paimon/core/realtime/arrow_realtime_store_factory.cpp b/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
index 4cfdb4c3d..babc55a3d 100644
--- a/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
+++ b/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
@@ -50,8 +50,11 @@ Result<std::shared_ptr<RealtimeStore>> ArrowRealtimeStoreFactory::Create(
                                                     request.memory_pool, arrow_pool);
     }
 
+    const PrimaryKeyRealtimeStoreCreateConfig& config =
+        std::get<PrimaryKeyRealtimeStoreCreateConfig>(request.mode_config);
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                           PrimaryKeyRealtimeStore::Create(imported_schema, request.memory_pool));
+                           PrimaryKeyRealtimeStore::Create(
+                               imported_schema, config.trimmed_primary_keys, request.memory_pool));
     return std::shared_ptr<RealtimeStore>(std::move(store));
 }
 
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index b99f67dd9..6b3afcd19 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -20,6 +20,7 @@
 
 #include <cstdint>
 #include <memory>
+#include <mutex>
 #include <optional>
 #include <unordered_map>
 #include <utility>
@@ -29,9 +30,11 @@
 #include "arrow/array/array_nested.h"
 #include "arrow/array/array_primitive.h"
 #include "arrow/array/builder_primitive.h"
+#include "arrow/buffer.h"
 #include "arrow/c/bridge.h"
 #include "arrow/compute/api.h"
 #include "arrow/type.h"
+#include "arrow/util/bit_util.h"
 #include "fmt/format.h"
 #include "paimon/common/data/columnar/columnar_batch_context.h"
 #include "paimon/common/data/columnar/columnar_row_ref.h"
@@ -42,6 +45,7 @@
 #include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/fields_comparator.h"
 #include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/realtime/realtime_fields.h"
 #include "paimon/core/utils/nested_projection_utils.h"
@@ -61,6 +65,68 @@ constexpr int32_t kPreparedValueStartIndex = 3;
 Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
     const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& read_type);
 
+class RealtimeOffsetCoverage {
+ public:
+    static Result<std::shared_ptr<RealtimeOffsetCoverage>> Create(
+        const OffsetRange& sealed_offsets, size_t reader_count,
+        const std::shared_ptr<arrow::MemoryPool>& arrow_pool) {
+        if (sealed_offsets.begin < 0 || sealed_offsets.end < sealed_offsets.begin) {
+            return Status::Invalid("PK real-time store returned an invalid sealed offset range");
+        }
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+            std::shared_ptr<arrow::Buffer> seen_offsets,
+            arrow::AllocateEmptyBitmap(sealed_offsets.Count(), arrow_pool.get()));
+        return std::shared_ptr<RealtimeOffsetCoverage>(new RealtimeOffsetCoverage(
+            sealed_offsets, reader_count, std::move(seen_offsets), arrow_pool));
+    }
+
+    Status Add(const arrow::Int64Array& offsets) {
+        std::lock_guard<std::mutex> lock(mutex_);
+        for (int64_t row = 0; row < offsets.length(); ++row) {
+            const int64_t offset = offsets.Value(row);
+            if (offset < sealed_offsets_.begin || offset >= sealed_offsets_.end) {
+                return Status::Invalid(
+                    "PK real-time store commit reader offset is outside the sealed range");
+            }
+            const int64_t index = offset - sealed_offsets_.begin;
+            if (arrow::bit_util::GetBit(seen_offsets_->data(), index)) {
+                return Status::Invalid(
+                    "PK real-time store commit readers contain duplicate REALTIME_OFFSET");
+            }
+            arrow::bit_util::SetBit(seen_offsets_->mutable_data(), index);
+            ++seen_count_;
+        }
+        return Status::OK();
+    }
+
+    Status FinishReader() {
+        std::lock_guard<std::mutex> lock(mutex_);
+        ++finished_reader_count_;
+        if (finished_reader_count_ == reader_count_ && seen_count_ != sealed_offsets_.Count()) {
+            return Status::Invalid(
+                "PK real-time store commit readers did not cover the sealed range");
+        }
+        return Status::OK();
+    }
+
+ private:
+    RealtimeOffsetCoverage(const OffsetRange& sealed_offsets, size_t reader_count,
+                           std::shared_ptr<arrow::Buffer> seen_offsets,
+                           const std::shared_ptr<arrow::MemoryPool>& arrow_pool)
+        : sealed_offsets_(sealed_offsets),
+          reader_count_(reader_count),
+          arrow_pool_(arrow_pool),
+          seen_offsets_(std::move(seen_offsets)) {}
+
+    OffsetRange sealed_offsets_;
+    size_t reader_count_;
+    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
+    std::shared_ptr<arrow::Buffer> seen_offsets_;
+    int64_t seen_count_ = 0;
+    size_t finished_reader_count_ = 0;
+    std::mutex mutex_;
+};
+
 Status CheckPreparedField(const std::shared_ptr<arrow::Schema>& schema, int32_t field_idx,
                           const DataField& expected_field) {
     if (schema->num_fields() <= field_idx) {
@@ -210,16 +276,20 @@ Result<std::shared_ptr<arrow::Array>> AlignStructArrayByPaimonIds(
             return Status::Invalid(
                 fmt::format("cannot find field id {} in prepared value struct", read_field_id));
         }
-        std::shared_ptr<arrow::Array> child = array->field(data_iter->second);
+        std::shared_ptr<arrow::Array> child =
+            arrow::MakeArray(array->data()->child_data[data_iter->second]);
         PAIMON_ASSIGN_OR_RAISE(child, AlignArrayByPaimonIds(child, read_field->type()));
         aligned_arrays.push_back(std::move(child));
     }
 
-    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-        std::shared_ptr<arrow::StructArray> aligned,
-        arrow::StructArray::Make(aligned_arrays, read_type->fields(), array->null_bitmap(),
-                                 array->null_count(), array->offset()));
-    return aligned;
+    std::shared_ptr<arrow::ArrayData> aligned_data = array->data()->Copy();
+    aligned_data->type = read_type;
+    aligned_data->child_data.clear();
+    aligned_data->child_data.reserve(aligned_arrays.size());
+    for (const std::shared_ptr<arrow::Array>& aligned_array : aligned_arrays) {
+        aligned_data->child_data.push_back(aligned_array->data());
+    }
+    return arrow::MakeArray(std::move(aligned_data));
 }
 
 Result<std::shared_ptr<arrow::Array>> AlignListArrayByPaimonIds(
@@ -351,15 +421,18 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
                            const std::optional<OffsetRange>& visible_offsets,
                            const std::shared_ptr<arrow::Schema>& key_schema,
                            const std::shared_ptr<arrow::Schema>& value_schema,
-                           const std::shared_ptr<MemoryPool>& pool, int64_t* raw_row_count)
+                           const std::shared_ptr<FieldsComparator>& key_comparator,
+                           const std::shared_ptr<MemoryPool>& pool,
+                           const std::shared_ptr<RealtimeOffsetCoverage>& offset_coverage)
         : reader_(std::move(reader)),
           prepared_schema_(prepared_schema),
           visible_offsets_(visible_offsets),
           key_schema_(key_schema),
           value_schema_(value_schema),
+          key_comparator_(key_comparator),
           pool_(pool),
           arrow_pool_(GetArrowPool(pool)),
-          raw_row_count_(raw_row_count) {}
+          offset_coverage_(offset_coverage) {}
 
     ~PreparedKeyValueReader() override {
         Close();
@@ -425,6 +498,10 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
             ResetBatchState();
             PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader_->NextBatch());
             if (BatchReader::IsEofBatch(batch)) {
+                if (offset_coverage_ && !offset_coverage_finished_) {
+                    offset_coverage_finished_ = true;
+                    PAIMON_RETURN_NOT_OK(offset_coverage_->FinishReader());
+                }
                 return std::unique_ptr<KeyValueRecordReader::Iterator>();
             }
             auto& [c_array, c_schema] = batch;
@@ -436,17 +513,14 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
             std::shared_ptr<arrow::StructArray> data_batch =
                 checked_pointer_cast<arrow::StructArray>(arrow_array);
             PAIMON_RETURN_NOT_OK(ValidatePreparedBatch(data_batch));
-            if (raw_row_count_ != nullptr) {
-                int64_t updated_count = 0;
-                if (__builtin_add_overflow(*raw_row_count_, data_batch->length(), &updated_count)) {
-                    return Status::Invalid("prepared raw row count overflow");
-                }
-                *raw_row_count_ = updated_count;
-            }
+            PAIMON_RETURN_NOT_OK(ValidateOrdering(data_batch));
 
             std::shared_ptr<arrow::NumericArray<arrow::Int64Type>> offset_array =
                 checked_pointer_cast<arrow::NumericArray<arrow::Int64Type>>(
                     data_batch->field(kRealtimeOffsetIndex));
+            if (offset_coverage_) {
+                PAIMON_RETURN_NOT_OK(offset_coverage_->Add(*offset_array));
+            }
             PAIMON_ASSIGN_OR_RAISE(
                 data_batch,
                 ApplyOffsetFilter(data_batch, offset_array, visible_offsets_, arrow_pool_.get()));
@@ -504,6 +578,36 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
         return Status::OK();
     }
 
+    Status ValidateOrdering(const std::shared_ptr<arrow::StructArray>& data_batch) {
+        if (data_batch->length() == 0) {
+            return Status::OK();
+        }
+        PAIMON_ASSIGN_OR_RAISE(arrow::ArrayVector key_fields,
+                               ProjectFieldsByPaimonIds(data_batch, prepared_schema_, key_schema_));
+        std::shared_ptr<ColumnarBatchContext> key_context =
+            std::make_shared<ColumnarBatchContext>(key_fields, pool_);
+        std::shared_ptr<arrow::Int64Array> sequences =
+            checked_pointer_cast<arrow::Int64Array>(data_batch->field(kSequenceNumberIndex));
+        for (int64_t row = 0; row < data_batch->length(); ++row) {
+            ColumnarRowRef current_key(key_context, row);
+            if (previous_key_context_) {
+                ColumnarRowRef previous_key(previous_key_context_, previous_key_row_);
+                const int32_t key_comparison =
+                    key_comparator_->CompareTo(previous_key, current_key);
+                if (key_comparison > 0 ||
+                    (key_comparison == 0 && previous_sequence_ > sequences->Value(row))) {
+                    return Status::Invalid(
+                        "PK real-time plugin reader is not globally sorted by primary key and "
+                        "sequence number");
+                }
+            }
+            previous_key_context_ = key_context;
+            previous_key_row_ = row;
+            previous_sequence_ = sequences->Value(row);
+        }
+        return Status::OK();
+    }
+
     void ResetBatchState() {
         key_ctx_.reset();
         value_ctx_.reset();
@@ -518,23 +622,32 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
     std::optional<OffsetRange> visible_offsets_;
     std::shared_ptr<arrow::Schema> key_schema_;
     std::shared_ptr<arrow::Schema> value_schema_;
+    std::shared_ptr<FieldsComparator> key_comparator_;
     std::shared_ptr<MemoryPool> pool_;
     std::shared_ptr<arrow::MemoryPool> arrow_pool_;
-    int64_t* raw_row_count_;
+    std::shared_ptr<RealtimeOffsetCoverage> offset_coverage_;
+    bool offset_coverage_finished_ = false;
     std::shared_ptr<ColumnarBatchContext> key_ctx_;
     std::shared_ptr<ColumnarBatchContext> value_ctx_;
     std::shared_ptr<arrow::NumericArray<arrow::Int8Type>> row_kind_array_;
     std::shared_ptr<arrow::NumericArray<arrow::Int64Type>> sequence_number_array_;
+    std::shared_ptr<ColumnarBatchContext> previous_key_context_;
+    int64_t previous_key_row_ = 0;
+    int64_t previous_sequence_ = 0;
 };
 
-}
+}  // namespace
 
-Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
+namespace {
+
+Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderImpl(
     std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
     const std::optional<OffsetRange>& visible_offsets,
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
-    const std::shared_ptr<MemoryPool>& memory_pool, int64_t* raw_row_count) {
+    const std::shared_ptr<FieldsComparator>& key_comparator,
+    const std::shared_ptr<MemoryPool>& memory_pool,
+    const std::shared_ptr<RealtimeOffsetCoverage>& offset_coverage) {
     std::unique_ptr<BatchReader> owned_reader = std::move(reader);
     if (!owned_reader) {
         return Status::Invalid("prepared batch reader cannot be null");
@@ -547,6 +660,9 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
     if (!value_schema) {
         return Status::Invalid("prepared value schema cannot be null");
     }
+    if (!key_comparator) {
+        return Status::Invalid("prepared key comparator cannot be null");
+    }
     if (!memory_pool) {
         return Status::Invalid("prepared reader memory pool cannot be null");
     }
@@ -555,11 +671,82 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
     if (!visible_offsets.has_value()) {
         PAIMON_RETURN_NOT_OK(ValidateExactCommitSchema(prepared_schema, value_schema));
     }
-    std::unique_ptr<KeyValueRecordReader> result(
-        new PreparedKeyValueReader(std::move(owned_reader), prepared_schema, visible_offsets,
-                                   key_schema, value_schema, memory_pool, raw_row_count));
+    std::unique_ptr<KeyValueRecordReader> result(new PreparedKeyValueReader(
+        std::move(owned_reader), prepared_schema, visible_offsets, key_schema, value_schema,
+        key_comparator, memory_pool, offset_coverage));
     close_guard.Release();
     return result;
 }
 
+}  // namespace
+
+Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
+    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::optional<OffsetRange>& visible_offsets,
+    const std::shared_ptr<arrow::Schema>& key_schema,
+    const std::shared_ptr<arrow::Schema>& value_schema,
+    const std::shared_ptr<FieldsComparator>& key_comparator,
+    const std::shared_ptr<MemoryPool>& memory_pool) {
+    return AdaptPreparedBatchReaderImpl(std::move(reader), prepared_schema, visible_offsets,
+                                        key_schema, value_schema, key_comparator, memory_pool,
+                                        /*offset_coverage=*/nullptr);
+}
+
+Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> AdaptPreparedCommitBatchReaders(
+    std::vector<std::unique_ptr<BatchReader>>&& readers,
+    const std::shared_ptr<arrow::Schema>& prepared_schema, const OffsetRange& sealed_offsets,
+    const std::shared_ptr<arrow::Schema>& key_schema,
+    const std::shared_ptr<arrow::Schema>& value_schema,
+    const std::shared_ptr<FieldsComparator>& key_comparator,
+    const std::shared_ptr<MemoryPool>& memory_pool) {
+    if (!memory_pool) {
+        return Status::Invalid("prepared reader memory pool cannot be null");
+    }
+    std::shared_ptr<arrow::MemoryPool> arrow_pool = GetArrowPool(memory_pool);
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<RealtimeOffsetCoverage> offset_coverage,
+        RealtimeOffsetCoverage::Create(sealed_offsets, readers.size(), arrow_pool));
+    ScopeGuard readers_guard([&readers]() {
+        for (const std::unique_ptr<BatchReader>& reader : readers) {
+            if (reader) {
+                reader->Close();
+            }
+        }
+    });
+    std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers;
+    ScopeGuard adapted_readers_guard([&adapted_readers]() {
+        for (const std::unique_ptr<KeyValueRecordReader>& reader : adapted_readers) {
+            reader->Close();
+        }
+    });
+    adapted_readers.reserve(readers.size());
+    for (std::unique_ptr<BatchReader>& reader : readers) {
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> adapted_reader,
+                               AdaptPreparedBatchReaderImpl(
+                                   std::move(reader), prepared_schema, std::nullopt, key_schema,
+                                   value_schema, key_comparator, memory_pool, offset_coverage));
+        adapted_readers.push_back(std::move(adapted_reader));
+    }
+    readers_guard.Release();
+    adapted_readers_guard.Release();
+    return adapted_readers;
+}
+
+Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
+    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::optional<OffsetRange>& visible_offsets,
+    const std::shared_ptr<arrow::Schema>& key_schema,
+    const std::shared_ptr<arrow::Schema>& value_schema,
+    const std::shared_ptr<MemoryPool>& memory_pool) {
+    if (!key_schema) {
+        return Status::Invalid("prepared key schema cannot be null");
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::vector<DataField> key_fields,
+                           DataField::ConvertArrowSchemaToDataFields(key_schema));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FieldsComparator> key_comparator,
+                           FieldsComparator::Create(key_fields, /*is_ascending_order=*/true));
+    return AdaptPreparedBatchReader(std::move(reader), prepared_schema, visible_offsets, key_schema,
+                                    value_schema, key_comparator, memory_pool);
 }
+
+}  // namespace paimon
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.h b/src/paimon/core/realtime/prepared_key_value_reader.h
index e7a6f9651..064a62958 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.h
+++ b/src/paimon/core/realtime/prepared_key_value_reader.h
@@ -21,6 +21,7 @@
 #include <cstdint>
 #include <memory>
 #include <optional>
+#include <vector>
 
 #include "arrow/type_fwd.h"
 #include "paimon/core/io/key_value_record_reader.h"
@@ -29,6 +30,7 @@
 
 namespace paimon {
 class BatchReader;
+class FieldsComparator;
 class MemoryPool;
 
 Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
@@ -36,6 +38,22 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
     const std::optional<OffsetRange>& visible_offsets,
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
-    const std::shared_ptr<MemoryPool>& memory_pool, int64_t* raw_row_count = nullptr);
+    const std::shared_ptr<FieldsComparator>& key_comparator,
+    const std::shared_ptr<MemoryPool>& memory_pool);
 
-}
+Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> AdaptPreparedCommitBatchReaders(
+    std::vector<std::unique_ptr<BatchReader>>&& readers,
+    const std::shared_ptr<arrow::Schema>& prepared_schema, const OffsetRange& sealed_offsets,
+    const std::shared_ptr<arrow::Schema>& key_schema,
+    const std::shared_ptr<arrow::Schema>& value_schema,
+    const std::shared_ptr<FieldsComparator>& key_comparator,
+    const std::shared_ptr<MemoryPool>& memory_pool);
+
+Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
+    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::optional<OffsetRange>& visible_offsets,
+    const std::shared_ptr<arrow::Schema>& key_schema,
+    const std::shared_ptr<arrow::Schema>& value_schema,
+    const std::shared_ptr<MemoryPool>& memory_pool);
+
+}  // namespace paimon
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index 0d6de9f5f..f43f60472 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -18,20 +18,31 @@
 
 #include "paimon/core/realtime/primary_key_realtime_store.h"
 
+#include <cstddef>
 #include <mutex>
+#include <optional>
 #include <utility>
+#include <vector>
 
 #include "arrow/api.h"
 #include "arrow/c/bridge.h"
+#include "arrow/compute/api.h"
+#include "paimon/common/data/columnar/columnar_batch_context.h"
+#include "paimon/common/data/columnar/columnar_row_ref.h"
 #include "paimon/common/metrics/metrics_impl.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/fields_comparator.h"
 #include "paimon/core/core_options.h"
+#include "paimon/core/index/pk/primary_key_index_definitions.h"
+#include "paimon/core/schema/table_schema.h"
 #include "paimon/macros.h"
 
 namespace paimon {
 
-Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options) {
+Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options, const TableSchema& schema) {
     if (options.GetBucket() <= 0) {
         return Status::NotImplemented("PK realtime v1 requires fixed buckets");
     }
@@ -60,6 +71,21 @@ Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options) {
         options.GetChangelogProducer() != ChangelogProducer::NONE) {
         return Status::NotImplemented("PK realtime v1 does not support lookup or early MOR");
     }
+    PAIMON_ASSIGN_OR_RAISE(std::vector<DataField> primary_key_fields,
+                           schema.TrimmedPrimaryKeyFields());
+    for (const DataField& field : primary_key_fields) {
+        if (field.Type()->id() == arrow::Type::FLOAT || field.Type()->id() == arrow::Type::DOUBLE) {
+            return Status::NotImplemented(
+                "PK realtime v1 does not support FLOAT or DOUBLE primary keys");
+        }
+    }
+    if (options.GlobalIndexEnabled()) {
+        PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions definitions,
+                               PrimaryKeyIndexDefinitions::Create(schema));
+        if (!definitions.Definitions().empty()) {
+            return Status::NotImplemented("PK realtime v1 does not support global indexes");
+        }
+    }
     return Status::OK();
 }
 
@@ -128,14 +154,56 @@ class ReadView final : public RealtimeReadView {
 
 class RawBatchReader final : public BatchReader {
  public:
-    RawBatchReader(std::vector<StoredBatch> batches)
-        : batches_(std::move(batches)), metrics_(std::make_shared<MetricsImpl>()) {}
+    RawBatchReader(std::vector<StoredBatch> batches, std::vector<int32_t> key_field_indexes,
+                   const std::shared_ptr<FieldsComparator>& key_comparator,
+                   const std::shared_ptr<MemoryPool>& memory_pool)
+        : batches_(std::move(batches)),
+          positions_(batches_.size(), 0),
+          key_field_indexes_(std::move(key_field_indexes)),
+          key_comparator_(key_comparator),
+          memory_pool_(memory_pool),
+          arrow_pool_(GetArrowPool(memory_pool)),
+          metrics_(std::make_shared<MetricsImpl>()) {
+        key_contexts_.reserve(batches_.size());
+        for (const StoredBatch& batch : batches_) {
+            arrow::ArrayVector key_arrays;
+            key_arrays.reserve(key_field_indexes_.size());
+            for (int32_t field_index : key_field_indexes_) {
+                key_arrays.push_back(batch.data->field(field_index));
+            }
+            key_contexts_.push_back(
+                std::make_shared<ColumnarBatchContext>(key_arrays, memory_pool_));
+        }
+    }
 
     Result<ReadBatch> NextBatch() override {
-        if (next_ == batches_.size()) {
+        if (closed_) {
+            return MakeEofBatch();
+        }
+        std::optional<size_t> selected;
+        for (size_t i = 0; i < batches_.size(); ++i) {
+            if (positions_[i] >= batches_[i].data->length()) {
+                continue;
+            }
+            if (!selected.has_value() || Less(i, selected.value())) {
+                selected = i;
+            }
+        }
+        if (!selected.has_value()) {
             return MakeEofBatch();
         }
-        const std::shared_ptr<arrow::StructArray>& batch = batches_[next_++].data;
+        const size_t batch_index = selected.value();
+        arrow::Int64Builder index_builder(arrow_pool_.get());
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(index_builder.Append(positions_[batch_index]));
+        std::shared_ptr<arrow::Array> index;
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(index_builder.Finish(&index));
+        arrow::compute::ExecContext context(arrow_pool_.get());
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+            arrow::Datum taken,
+            arrow::compute::Take(arrow::Datum(batches_[batch_index].data), arrow::Datum(index),
+                                 arrow::compute::TakeOptions::NoBoundsCheck(), &context));
+        std::shared_ptr<arrow::Array> batch = taken.make_array();
+        ++positions_[batch_index];
         auto array = std::make_unique<ArrowArray>();
         auto schema = std::make_unique<ArrowSchema>();
         PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*batch, array.get(), schema.get()));
@@ -146,12 +214,38 @@ class RawBatchReader final : public BatchReader {
         return metrics_;
     }
     void Close() override {
+        if (closed_) {
+            return;
+        }
+        closed_ = true;
         batches_.clear();
+        positions_.clear();
+        key_contexts_.clear();
     }
 
  private:
+    bool Less(size_t left, size_t right) const {
+        ColumnarRowRef left_key(key_contexts_[left], positions_[left]);
+        ColumnarRowRef right_key(key_contexts_[right], positions_[right]);
+        const int32_t key_comparison = key_comparator_->CompareTo(left_key, right_key);
+        if (key_comparison != 0) {
+            return key_comparison < 0;
+        }
+        const std::shared_ptr<arrow::Int64Array> left_sequences =
+            checked_pointer_cast<arrow::Int64Array>(batches_[left].data->field(1));
+        const std::shared_ptr<arrow::Int64Array> right_sequences =
+            checked_pointer_cast<arrow::Int64Array>(batches_[right].data->field(1));
+        return left_sequences->Value(positions_[left]) < right_sequences->Value(positions_[right]);
+    }
+
+    bool closed_ = false;
     std::vector<StoredBatch> batches_;
-    size_t next_ = 0;
+    std::vector<int64_t> positions_;
+    std::vector<int32_t> key_field_indexes_;
+    std::shared_ptr<FieldsComparator> key_comparator_;
+    std::shared_ptr<MemoryPool> memory_pool_;
+    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
+    std::vector<std::shared_ptr<ColumnarBatchContext>> key_contexts_;
     std::shared_ptr<Metrics> metrics_;
 };
 
@@ -159,8 +253,13 @@ class RawBatchReader final : public BatchReader {
 
 class PrimaryKeyRealtimeStore::Impl {
  public:
-    explicit Impl(std::shared_ptr<arrow::Schema> prepared_schema)
-        : prepared_schema_(std::move(prepared_schema)) {}
+    Impl(std::shared_ptr<arrow::Schema> prepared_schema, std::vector<int32_t> key_field_indexes,
+         const std::shared_ptr<FieldsComparator>& key_comparator,
+         const std::shared_ptr<MemoryPool>& memory_pool)
+        : prepared_schema_(std::move(prepared_schema)),
+          key_field_indexes_(std::move(key_field_indexes)),
+          key_comparator_(key_comparator),
+          memory_pool_(memory_pool) {}
 
     Status Write(RealtimeWriteBatch&& write_batch) {
         if (!write_batch.batch || !write_batch.batch->GetData()) {
@@ -212,9 +311,9 @@ class PrimaryKeyRealtimeStore::Impl {
             return Status::Invalid("segment was not created by the PK real-time store");
         }
         std::vector<std::unique_ptr<BatchReader>> readers;
-        readers.reserve(segment->Batches().size());
-        for (const StoredBatch& batch : segment->Batches()) {
-            readers.push_back(std::make_unique<RawBatchReader>(std::vector<StoredBatch>{batch}));
+        if (!segment->Batches().empty()) {
+            readers.push_back(std::make_unique<RawBatchReader>(
+                segment->Batches(), key_field_indexes_, key_comparator_, memory_pool_));
         }
         return readers;
     }
@@ -238,16 +337,13 @@ class PrimaryKeyRealtimeStore::Impl {
             return Status::Invalid("read view was not created by the PK real-time store");
         }
         std::vector<std::unique_ptr<BatchReader>> readers;
-        size_t batch_count = 0;
+        std::vector<StoredBatch> batches;
         for (const std::shared_ptr<Segment>& segment : typed->Segments()) {
-            batch_count += segment->Batches().size();
+            batches.insert(batches.end(), segment->Batches().begin(), segment->Batches().end());
         }
-        readers.reserve(batch_count);
-        for (const std::shared_ptr<Segment>& segment : typed->Segments()) {
-            for (const StoredBatch& batch : segment->Batches()) {
-                readers.push_back(
-                    std::make_unique<RawBatchReader>(std::vector<StoredBatch>{batch}));
-            }
+        if (!batches.empty()) {
+            readers.push_back(std::make_unique<RawBatchReader>(
+                std::move(batches), key_field_indexes_, key_comparator_, memory_pool_));
         }
         return readers;
     }
@@ -273,6 +369,9 @@ class PrimaryKeyRealtimeStore::Impl {
 
  private:
     std::shared_ptr<arrow::Schema> prepared_schema_;
+    std::vector<int32_t> key_field_indexes_;
+    std::shared_ptr<FieldsComparator> key_comparator_;
+    std::shared_ptr<MemoryPool> memory_pool_;
     mutable std::mutex mutex_;
     std::vector<StoredBatch> building_;
     std::vector<std::shared_ptr<Segment>> sealed_;
@@ -286,12 +385,29 @@ PrimaryKeyRealtimeStore::~PrimaryKeyRealtimeStore() = default;
 
 Result<std::shared_ptr<PrimaryKeyRealtimeStore>> PrimaryKeyRealtimeStore::Create(
     const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::vector<std::string>& trimmed_primary_keys,
     const std::shared_ptr<MemoryPool>& memory_pool) {
-    if (!prepared_schema || !memory_pool) {
+    if (!prepared_schema || trimmed_primary_keys.empty() || !memory_pool) {
         return Status::Invalid("PK prepared schema or memory pool is null");
     }
-    return std::shared_ptr<PrimaryKeyRealtimeStore>(
-        new PrimaryKeyRealtimeStore(std::make_unique<Impl>(prepared_schema)));
+    std::vector<int32_t> key_field_indexes;
+    std::vector<DataField> key_fields;
+    key_field_indexes.reserve(trimmed_primary_keys.size());
+    key_fields.reserve(trimmed_primary_keys.size());
+    for (const std::string& key : trimmed_primary_keys) {
+        const int32_t field_index = prepared_schema->GetFieldIndex(key);
+        if (field_index < 3) {
+            return Status::Invalid("PK field is missing from prepared schema: ", key);
+        }
+        key_field_indexes.push_back(field_index);
+        PAIMON_ASSIGN_OR_RAISE(DataField field, DataField::ConvertArrowFieldToDataField(
+                                                    prepared_schema->field(field_index)));
+        key_fields.push_back(std::move(field));
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FieldsComparator> key_comparator,
+                           FieldsComparator::Create(key_fields, /*is_ascending_order=*/true));
+    return std::shared_ptr<PrimaryKeyRealtimeStore>(new PrimaryKeyRealtimeStore(
+        std::make_unique<Impl>(prepared_schema, key_field_indexes, key_comparator, memory_pool)));
 }
 Status PrimaryKeyRealtimeStore::Write(RealtimeWriteBatch&& batch) {
     return impl_->Write(std::move(batch));
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.h b/src/paimon/core/realtime/primary_key_realtime_store.h
index 5e18dd74f..d6a23ccf9 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.h
+++ b/src/paimon/core/realtime/primary_key_realtime_store.h
@@ -31,14 +31,16 @@ namespace paimon {
 
 class CoreOptions;
 class MemoryPool;
+class TableSchema;
 
-Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options);
+Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options, const TableSchema& schema);
 
 /// In-memory store for prepared primary-key real-time batches.
 class PrimaryKeyRealtimeStore final : public RealtimeStore {
  public:
     static Result<std::shared_ptr<PrimaryKeyRealtimeStore>> Create(
         const std::shared_ptr<arrow::Schema>& prepared_schema,
+        const std::vector<std::string>& trimmed_primary_keys,
         const std::shared_ptr<MemoryPool>& memory_pool);
 
     ~PrimaryKeyRealtimeStore() override;
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index 43831d7be..cafe3682e 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -31,6 +31,7 @@
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/realtime/realtime_fields.h"
+#include "paimon/core/schema/table_schema.h"
 #include "paimon/macros.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/testing/utils/testharness.h"
@@ -44,7 +45,33 @@ std::shared_ptr<arrow::Schema> PreparedSchema() {
          DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
              ->WithNullable(false),
          DataField::ConvertDataFieldToArrowField(RealtimeOffsetField())->WithNullable(false),
-         arrow::field("id", arrow::int64()), arrow::field("value", arrow::utf8())});
+         DataField::ConvertDataFieldToArrowField(DataField(0, arrow::field("id", arrow::int64()))),
+         DataField::ConvertDataFieldToArrowField(
+             DataField(1, arrow::field("value", arrow::utf8())))});
+}
+
+std::shared_ptr<arrow::Schema> NestedPreparedSchema() {
+    return arrow::schema(
+        {DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
+         DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
+             ->WithNullable(false),
+         DataField::ConvertDataFieldToArrowField(RealtimeOffsetField())->WithNullable(false),
+         DataField::ConvertDataFieldToArrowField(DataField(0, arrow::field("id", arrow::int64()))),
+         DataField::ConvertDataFieldToArrowField(DataField(
+             1,
+             arrow::field("value",
+                          arrow::struct_({arrow::field("name", arrow::utf8()),
+                                          arrow::field("items", arrow::list(arrow::int32()))}))))});
+}
+
+std::shared_ptr<TableSchema> PkSchema(
+    const std::shared_ptr<arrow::DataType>& key_type = arrow::int64(),
+    const std::map<std::string, std::string>& options = {}) {
+    return TableSchema::Create(
+               /*schema_id=*/0,
+               arrow::schema({arrow::field("id", key_type), arrow::field("value", arrow::utf8())}),
+               /*partition_keys=*/{}, /*primary_keys=*/{"id"}, options)
+        .value();
 }
 
 std::unique_ptr<RecordBatch> MakeBatch(const std::string& json) {
@@ -56,6 +83,27 @@ std::unique_ptr<RecordBatch> MakeBatch(const std::string& json) {
     return RecordBatchBuilder(c_array.get()).Finish().value();
 }
 
+std::unique_ptr<RecordBatch> MakeBatch(const std::shared_ptr<arrow::Schema>& schema,
+                                       const std::string& json) {
+    std::shared_ptr<arrow::Array> array =
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema->fields()), json)
+            .ValueOrDie();
+    auto c_array = std::make_unique<ArrowArray>();
+    EXPECT_TRUE(arrow::ExportArray(*array, c_array.get()).ok());
+    return RecordBatchBuilder(c_array.get()).Finish().value();
+}
+
+void AssertOffsetsZero(const ArrowArray* array) {
+    ASSERT_NE(nullptr, array);
+    ASSERT_EQ(0, array->offset);
+    for (int64_t child = 0; child < array->n_children; ++child) {
+        AssertOffsetsZero(array->children[child]);
+    }
+    if (array->dictionary) {
+        AssertOffsetsZero(array->dictionary);
+    }
+}
+
 Result<std::string> ReadJson(const std::vector<std::unique_ptr<BatchReader>>& readers) {
     std::vector<std::shared_ptr<arrow::Array>> batches;
     for (const std::unique_ptr<BatchReader>& reader : readers) {
@@ -77,7 +125,7 @@ Result<std::string> ReadJson(const std::vector<std::unique_ptr<BatchReader>>& re
 
 TEST(PrimaryKeyRealtimeStoreOptionsTest, TestSupportedOptions) {
     ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
-    ASSERT_OK(ValidatePrimaryKeyRealtimeOptions(options));
+    ASSERT_OK(ValidatePrimaryKeyRealtimeOptions(options, *PkSchema()));
 }
 
 TEST(PrimaryKeyRealtimeStoreOptionsTest, TestUnsupportedOptions) {
@@ -95,13 +143,31 @@ TEST(PrimaryKeyRealtimeStoreOptionsTest, TestUnsupportedOptions) {
     };
     for (const std::map<std::string, std::string>& option_map : unsupported_options) {
         ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
-        ASSERT_NOK(ValidatePrimaryKeyRealtimeOptions(options));
+        ASSERT_NOK(ValidatePrimaryKeyRealtimeOptions(options, *PkSchema()));
     }
 }
 
+TEST(PrimaryKeyRealtimeStoreOptionsTest, TestRejectsFloatingPrimaryKeys) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
+    ASSERT_NOK_WITH_MSG(ValidatePrimaryKeyRealtimeOptions(options, *PkSchema(arrow::float32())),
+                        "FLOAT or DOUBLE primary keys");
+    ASSERT_NOK_WITH_MSG(ValidatePrimaryKeyRealtimeOptions(options, *PkSchema(arrow::float64())),
+                        "FLOAT or DOUBLE primary keys");
+}
+
+TEST(PrimaryKeyRealtimeStoreOptionsTest, TestRejectsEnabledGlobalIndex) {
+    const std::map<std::string, std::string> option_map = {{Options::BUCKET, "1"},
+                                                           {Options::PK_BTREE_INDEX_COLUMNS, "id"}};
+    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
+    ASSERT_NOK_WITH_MSG(
+        ValidatePrimaryKeyRealtimeOptions(options, *PkSchema(arrow::int64(), option_map)),
+        "does not support global indexes");
+}
+
 TEST(PrimaryKeyRealtimeStoreTest, TestWriteAndSealValidation) {
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(PreparedSchema(), GetDefaultPool()));
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr<PrimaryKeyRealtimeStore> store,
+        PrimaryKeyRealtimeStore::Create(PreparedSchema(), {"id"}, GetDefaultPool()));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
                          store->SealForCommit());
     ASSERT_FALSE(segment.has_value());
@@ -131,10 +197,11 @@ TEST(PrimaryKeyRealtimeStoreTest, TestWriteAndSealValidation) {
 }
 
 TEST(PrimaryKeyRealtimeStoreTest, TestCommitBatches) {
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(PreparedSchema(), GetDefaultPool()));
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr<PrimaryKeyRealtimeStore> store,
+        PrimaryKeyRealtimeStore::Create(PreparedSchema(), {"id"}, GetDefaultPool()));
     ASSERT_OK(store->Write(RealtimeWriteBatch{
-        MakeBatch(R"([[0, 5, 0, 3, "three"], [1, 6, 1, 1, "before"]])"), OffsetRange(0, 2)}));
+        MakeBatch(R"([[1, 6, 1, 1, "before"], [0, 5, 0, 3, "three"]])"), OffsetRange(0, 2)}));
     ASSERT_OK(store->Write(
         RealtimeWriteBatch{MakeBatch(R"([[2, 7, 2, 2, "after"]])"), OffsetRange(2, 3)}));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
@@ -142,18 +209,45 @@ TEST(PrimaryKeyRealtimeStoreTest, TestCommitBatches) {
     ASSERT_TRUE(segment.has_value());
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
                          store->CreateCommitReaders(segment.value()));
+    ASSERT_EQ(1, readers.size());
     ASSERT_OK_AND_ASSIGN(std::string actual, ReadJson(readers));
     ASSERT_EQ(
-        "-- is_valid: all not null\n-- child 0 type: int8\n  [\n    0,\n    1,\n    2\n  ]\n-- "
-        "child 1 type: int64\n  [\n    5,\n    6,\n    7\n  ]\n-- child 2 type: int64\n  [\n    "
-        "0,\n    1,\n    2\n  ]\n-- child 3 type: int64\n  [\n    3,\n    1,\n    2\n  ]\n-- child "
-        "4 type: string\n  [\n    \"three\",\n    \"before\",\n    \"after\"\n  ]",
+        "-- is_valid: all not null\n-- child 0 type: int8\n  [\n    1,\n    2,\n    0\n  ]\n-- "
+        "child 1 type: int64\n  [\n    6,\n    7,\n    5\n  ]\n-- child 2 type: int64\n  [\n    "
+        "1,\n    2,\n    0\n  ]\n-- child 3 type: int64\n  [\n    1,\n    2,\n    3\n  ]\n-- child "
+        "4 type: string\n  [\n    \"before\",\n    \"after\",\n    \"three\"\n  ]",
         actual);
+    readers[0]->Close();
+    readers[0]->Close();
+    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch eof, readers[0]->NextBatch());
+    ASSERT_TRUE(BatchReader::IsEofBatch(eof));
 }
 
-TEST(PrimaryKeyRealtimeStoreTest, TestReclaimKeepsReadView) {
+TEST(PrimaryKeyRealtimeStoreTest, TestCommitReaderExportsZeroOffsets) {
+    std::shared_ptr<arrow::Schema> schema = NestedPreparedSchema();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(PreparedSchema(), GetDefaultPool()));
+                         PrimaryKeyRealtimeStore::Create(schema, {"id"}, GetDefaultPool()));
+    ASSERT_OK(store->Write(RealtimeWriteBatch{
+        MakeBatch(schema, R"([[0, 1, 0, 1, ["one", [1, 2]]], [0, 2, 1, 2, ["two", [3, 4]]]])"),
+        OffsetRange(0, 2)}));
+    ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
+                         store->SealForCommit());
+    ASSERT_TRUE(segment.has_value());
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store->CreateCommitReaders(segment.value()));
+    ASSERT_EQ(1, readers.size());
+    for (int32_t row = 0; row < 2; ++row) {
+        ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, readers[0]->NextBatch());
+        ASSERT_FALSE(BatchReader::IsEofBatch(batch));
+        AssertOffsetsZero(batch.first.get());
+        ASSERT_TRUE(arrow::ImportArray(batch.first.get(), batch.second.get()).ok());
+    }
+}
+
+TEST(PrimaryKeyRealtimeStoreTest, TestReclaimKeepsReadView) {
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr<PrimaryKeyRealtimeStore> store,
+        PrimaryKeyRealtimeStore::Create(PreparedSchema(), {"id"}, GetDefaultPool()));
     ASSERT_OK(
         store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 0, 4, 1, "one"]])"), OffsetRange(4, 5)}));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
@@ -163,5 +257,27 @@ TEST(PrimaryKeyRealtimeStoreTest, TestReclaimKeepsReadView) {
     ASSERT_EQ(std::optional<OffsetRange>(OffsetRange(4, 5)), view->GetOffsetRange());
 }
 
+TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderCardinalityIsConstant) {
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr<PrimaryKeyRealtimeStore> store,
+        PrimaryKeyRealtimeStore::Create(PreparedSchema(), {"id"}, GetDefaultPool()));
+    ASSERT_OK(
+        store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 1, 0, 2, "two"]])"), OffsetRange(0, 1)}));
+    ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
+                         store->SealForCommit());
+    ASSERT_TRUE(segment.has_value());
+    ASSERT_OK(
+        store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 2, 1, 1, "one"]])"), OffsetRange(1, 2)}));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
+    RealtimeQueryContext context{/*read_schema=*/nullptr, /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store->CreateQueryReaders(view, /*offset_begin=*/0, context));
+    ASSERT_EQ(1, readers.size());
+    ASSERT_OK_AND_ASSIGN(std::string actual, ReadJson(readers));
+    ASSERT_NE(std::string::npos, actual.find("\"one\""));
+    ASSERT_NE(std::string::npos, actual.find("\"two\""));
+}
+
 }  // namespace
 }  // namespace paimon::test
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index 415052a69..215e066ee 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -34,14 +34,33 @@
 #include <utility>
 #include <vector>
 
+#include "arrow/api.h"
+#include "arrow/c/bridge.h"
 #include "arrow/c/helpers.h"
 #include "paimon/arrow/abi.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/scope_guard.h"
 #include "paimon/common/utils/uuid.h"
 #include "paimon/macros.h"
 #include "paimon/realtime/realtime_store.h"
 #include "paimon/status.h"
 
 namespace paimon {
+namespace {
+
+bool SameMode(const RealtimeStoreCreateConfig& left, const RealtimeStoreCreateConfig& right) {
+    if (left.index() != right.index()) {
+        return false;
+    }
+    if (const auto* left_pk = std::get_if<PrimaryKeyRealtimeStoreCreateConfig>(&left)) {
+        const auto& right_pk = std::get<PrimaryKeyRealtimeStoreCreateConfig>(right);
+        return left_pk->trimmed_primary_keys == right_pk.trimmed_primary_keys;
+    }
+    return true;
+}
+
+}  // namespace
+
 RealtimeContextImpl::RealtimeContextImpl(const std::shared_ptr<RealtimeStoreFactory>& factory)
     : factory_(factory) {}
 
@@ -78,6 +97,14 @@ Status RealtimeContextImpl::Start() {
 
 Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     RealtimeStoreCreateRequest&& request) {
+    if (!request.write_schema || !request.write_schema->release) {
+        return Status::Invalid("real-time store write schema is null");
+    }
+    ScopeGuard schema_guard(
+        [schema = request.write_schema.get()]() { ArrowSchemaRelease(schema); });
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> requested_schema,
+                                      arrow::ImportSchema(request.write_schema.get()));
+    schema_guard.Release();
     std::lock_guard<std::mutex> progress_lock(progress_mutex_);
     std::lock_guard<std::mutex> registry_lock(mutex_);
     const RealtimePartitionBucket key(request.partition, request.bucket);
@@ -86,19 +113,18 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     auto offset_iter = committed_offsets_.find(key);
     if (offset_iter != committed_offsets_.end()) {
         if (offset_iter->second == std::numeric_limits<int64_t>::max()) {
-            if (request.write_schema) {
-                ArrowSchemaRelease(request.write_schema.get());
-            }
             return Status::Invalid("real-time offset has reached INT64_MAX");
         }
         initial_offset = offset_iter->second;
     }
     if (iter != stores_.end()) {
-        if (request.write_schema) {
-            ArrowSchemaRelease(request.write_schema.get());
+        if (!SameMode(iter->second.mode_config, request.mode_config) ||
+            !iter->second.write_schema->Equals(*requested_schema, /*check_metadata=*/true)) {
+            return Status::Invalid(
+                "real-time store schema or mode does not match the registered store");
         }
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeReadView> read_view,
-                               iter->second->AcquireReadView());
+                               iter->second.store->AcquireReadView());
         if (!read_view) {
             return Status::Invalid("real-time store returned a null read view");
         }
@@ -113,17 +139,18 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
                 initial_offset = memory_range->end;
             }
         }
-        return RealtimeStoreState{iter->second, initial_offset};
+        return RealtimeStoreState{iter->second.store, initial_offset};
     }
     if (!request.memory_pool) {
-        if (request.write_schema) {
-            ArrowSchemaRelease(request.write_schema.get());
-        }
         return Status::Invalid("real-time store memory pool is null");
     }
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(
+        arrow::ExportSchema(*requested_schema, request.write_schema.get()));
+    RealtimeStoreCreateConfig mode_config = request.mode_config;
     Result<std::shared_ptr<RealtimeStore>> store_result = factory_->Create(std::move(request));
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> store, std::move(store_result));
-    stores_.emplace(key, store);
+    stores_.emplace(key,
+                    RealtimeStoreRegistryEntry{store, requested_schema, std::move(mode_config)});
     if (offset_iter != committed_offsets_.end()) {
         reclaimed_offsets_.emplace(key, offset_iter->second);
     }
@@ -147,9 +174,9 @@ Result<std::vector<RealtimePartitionBucketView>> RealtimeContextImpl::AcquireRea
     result.reserve(stores_.size());
     for (const auto& [partition_bucket, store] : stores_) {
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeReadView> read_view,
-                               store->AcquireReadView());
+                               store.store->AcquireReadView());
         result.push_back(
-            RealtimePartitionBucketView{partition_bucket, store, std::move(read_view)});
+            RealtimePartitionBucketView{partition_bucket, store.store, std::move(read_view)});
     }
     return result;
 }
@@ -266,7 +293,7 @@ Status RealtimeContextImpl::AdvanceCommittedProgress(int64_t snapshot_id,
             }
             auto store_iter = stores_.find(partition_bucket);
             if (store_iter != stores_.end()) {
-                notifications.emplace_back(partition_bucket, store_iter->second,
+                notifications.emplace_back(partition_bucket, store_iter->second.store,
                                            committed_end_offset);
             }
         }
diff --git a/src/paimon/core/realtime/realtime_context_impl.h b/src/paimon/core/realtime/realtime_context_impl.h
index aa4d263c6..9fa145e99 100644
--- a/src/paimon/core/realtime/realtime_context_impl.h
+++ b/src/paimon/core/realtime/realtime_context_impl.h
@@ -38,6 +38,10 @@
 
 struct ArrowSchema;
 
+namespace arrow {
+class Schema;
+}  // namespace arrow
+
 namespace paimon {
 
 class RealtimeStore;
@@ -55,6 +59,12 @@ struct RealtimePartitionBucketView {
     std::shared_ptr<RealtimeReadView> read_view;
 };
 
+struct RealtimeStoreRegistryEntry {
+    std::shared_ptr<RealtimeStore> store;
+    std::shared_ptr<arrow::Schema> write_schema;
+    RealtimeStoreCreateConfig mode_config;
+};
+
 class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
  public:
     static Result<std::shared_ptr<RealtimeContextImpl>> Create(
@@ -98,7 +108,7 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
     std::shared_ptr<RealtimeStoreFactory> factory_;
     std::mutex mutex_;
     std::mutex progress_mutex_;
-    std::map<RealtimePartitionBucket, std::shared_ptr<RealtimeStore>> stores_;
+    std::map<RealtimePartitionBucket, RealtimeStoreRegistryEntry> stores_;
     std::map<RealtimePartitionBucket, int64_t> materialized_max_sequence_numbers_;
     // Full-table progress used as the initial offset when a store is created lazily.
     RealtimeOffsetMap committed_offsets_;
diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp
index 15066ca1d..2b47e9dc9 100644
--- a/src/paimon/core/realtime/realtime_context_test.cpp
+++ b/src/paimon/core/realtime/realtime_context_test.cpp
@@ -96,10 +96,12 @@ class TestingRealtimeStoreFactory : public RealtimeStoreFactory {
     std::vector<std::shared_ptr<TestingRealtimeStore>> stores;
 };
 
-std::unique_ptr<ArrowSchema> MakeWriteSchema() {
+std::unique_ptr<ArrowSchema> MakeWriteSchema(
+    const std::shared_ptr<arrow::DataType>& id_type = arrow::int64(),
+    const std::shared_ptr<arrow::KeyValueMetadata>& metadata = nullptr) {
     auto schema = std::make_unique<ArrowSchema>();
     EXPECT_TRUE(
-        arrow::ExportSchema(*arrow::schema({arrow::field("id", arrow::int64())}), schema.get())
+        arrow::ExportSchema(*arrow::schema({arrow::field("id", id_type)}, metadata), schema.get())
             .ok());
     return schema;
 }
@@ -158,6 +160,27 @@ TEST(RealtimeContextTest, TestReusesStoreAndCapturesRegisteredViews) {
     ASSERT_EQ(1, factory->stores[2]->acquire_count);
 }
 
+TEST(RealtimeContextTest, TestRejectsMismatchedSchemaOnStoreReuse) {
+    auto factory = std::make_shared<TestingRealtimeStoreFactory>();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
+    const std::map<std::string, std::string> partition = {{"dt", "2026-08-02"}};
+    std::shared_ptr<arrow::KeyValueMetadata> metadata =
+        arrow::key_value_metadata({"identity"}, {"v1"});
+    ASSERT_OK(GetOrCreateAppendStore(
+        context, partition, 0, MakeWriteSchema(arrow::int64(), metadata), {}, GetDefaultPool()));
+    ASSERT_NOK_WITH_MSG(
+        GetOrCreateAppendStore(context, partition, 0, MakeWriteSchema(arrow::int32(), metadata), {},
+                               GetDefaultPool()),
+        "schema or mode does not match");
+    ASSERT_NOK_WITH_MSG(
+        GetOrCreateAppendStore(
+            context, partition, 0,
+            MakeWriteSchema(arrow::int64(), arrow::key_value_metadata({"identity"}, {"v2"})), {},
+            GetDefaultPool()),
+        "schema or mode does not match");
+    ASSERT_EQ(1, factory->stores.size());
+}
+
 TEST(RealtimeContextTest, TestReconcilesPrimaryKeyInitialSequence) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index b53831f0a..82318eadb 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -266,15 +266,12 @@ Result<CommitIncrement> RealtimePrimaryKeyWriter::PrepareCommit(bool wait_compac
         return Status::Invalid("PK real-time store sealed a null segment");
     }
     std::optional<OffsetRange> sealed_range;
-    int64_t expected_raw_row_count = 0;
     if (segment) {
         sealed_range = segment.value()->GetOffsetRange();
-        if (sealed_range->begin < 0 || sealed_range->end < sealed_range->begin ||
-            __builtin_sub_overflow(sealed_range->end, sealed_range->begin,
-                                   &expected_raw_row_count)) {
+        if (sealed_range->begin < 0 || sealed_range->end < sealed_range->begin) {
             return Status::Invalid("PK real-time store returned an invalid sealed offset range");
         }
-        PAIMON_RETURN_NOT_OK(FlushSegment(segment.value(), expected_raw_row_count));
+        PAIMON_RETURN_NOT_OK(FlushSegment(segment.value(), sealed_range.value()));
     }
     PAIMON_ASSIGN_OR_RAISE(CommitIncrement increment,
                            merge_tree_writer_->PrepareCommit(wait_compaction));
@@ -285,7 +282,7 @@ Result<CommitIncrement> RealtimePrimaryKeyWriter::PrepareCommit(bool wait_compac
 }
 
 Status RealtimePrimaryKeyWriter::FlushSegment(const std::shared_ptr<RealtimeSegmentHandle>& segment,
-                                              int64_t expected_raw_row_count) {
+                                              const OffsetRange& sealed_offsets) {
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
                            realtime_store_->CreateCommitReaders(segment));
     ScopeGuard readers_guard([&readers]() {
@@ -295,28 +292,25 @@ Status RealtimePrimaryKeyWriter::FlushSegment(const std::shared_ptr<RealtimeSegm
             }
         }
     });
-    int64_t raw_row_count = 0;
-    std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
-    sorted_readers.reserve(readers.size());
-    for (std::unique_ptr<BatchReader>& reader : readers) {
+    for (const std::unique_ptr<BatchReader>& reader : readers) {
         if (!reader) {
             return Status::Invalid("PK real-time store returned a null commit reader");
         }
-        PAIMON_ASSIGN_OR_RAISE(
-            std::unique_ptr<KeyValueRecordReader> prepared_reader,
-            AdaptPreparedBatchReader(std::move(reader), prepared_schema_, std::nullopt, key_schema_,
-                                     write_schema_, memory_pool_, &raw_row_count));
+    }
+    PAIMON_ASSIGN_OR_RAISE(
+        std::vector<std::unique_ptr<KeyValueRecordReader>> prepared_readers,
+        AdaptPreparedCommitBatchReaders(std::move(readers), prepared_schema_, sealed_offsets,
+                                        key_schema_, write_schema_, key_comparator_, memory_pool_));
+    std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
+    sorted_readers.reserve(prepared_readers.size());
+    for (std::unique_ptr<KeyValueRecordReader>& prepared_reader : prepared_readers) {
         auto merge_function = std::make_unique<DeduplicateMergeFunction>(/*ignore_delete=*/false);
         sorted_readers.push_back(std::make_unique<MergedKeyValueRecordReader>(
             std::move(prepared_reader), key_comparator_,
             std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge_function))));
     }
     readers_guard.Release();
-    PAIMON_RETURN_NOT_OK(merge_tree_writer_->WriteSortedReaders(std::move(sorted_readers)));
-    if (raw_row_count != expected_raw_row_count) {
-        return Status::Invalid("PK real-time store commit readers did not cover the sealed range");
-    }
-    return Status::OK();
+    return merge_tree_writer_->WriteSortedReaders(std::move(sorted_readers));
 }
 
 Status RealtimePrimaryKeyWriter::Compact(bool) {
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.h b/src/paimon/core/realtime/realtime_primary_key_writer.h
index 9a5aa4c68..2eaf7ce24 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.h
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.h
@@ -79,7 +79,7 @@ class RealtimePrimaryKeyWriter final : public BatchWriter {
                              const std::shared_ptr<MemoryPool>& memory_pool);
 
     Status FlushSegment(const std::shared_ptr<RealtimeSegmentHandle>& segment,
-                        int64_t expected_raw_row_count);
+                        const OffsetRange& sealed_offsets);
 
     std::shared_ptr<MemoryPool> memory_pool_;
     std::shared_ptr<arrow::MemoryPool> arrow_pool_;
diff --git a/src/paimon/core/realtime/realtime_reader.h b/src/paimon/core/realtime/realtime_reader.h
index 6c25fd853..a041e0caa 100644
--- a/src/paimon/core/realtime/realtime_reader.h
+++ b/src/paimon/core/realtime/realtime_reader.h
@@ -44,10 +44,16 @@ class RealtimeReader final : public BatchReader {
     }
 
     Result<ReadBatch> NextBatch() override {
+        if (closed_) {
+            return MakeEofBatch();
+        }
         return reader_->NextBatch();
     }
 
     Result<ReadBatchWithBitmap> NextBatchWithBitmap() override {
+        if (closed_) {
+            return MakeEofBatchWithBitmap();
+        }
         return reader_->NextBatchWithBitmap();
     }
 
@@ -56,6 +62,10 @@ class RealtimeReader final : public BatchReader {
     }
 
     void Close() override {
+        if (closed_) {
+            return;
+        }
+        closed_ = true;
         reader_->Close();
         read_view_.reset();
     }
@@ -68,6 +78,7 @@ class RealtimeReader final : public BatchReader {
     // before releasing the data it references.
     std::shared_ptr<RealtimeReadView> read_view_;
     std::unique_ptr<BatchReader> reader_;
+    bool closed_ = false;
 };
 
 }  // namespace paimon
diff --git a/src/paimon/core/realtime/realtime_reader_test.cpp b/src/paimon/core/realtime/realtime_reader_test.cpp
index ec37cfed4..10f6ce5be 100644
--- a/src/paimon/core/realtime/realtime_reader_test.cpp
+++ b/src/paimon/core/realtime/realtime_reader_test.cpp
@@ -37,6 +37,8 @@ class TestingReadView : public RealtimeReadView {
 
 class TestingBatchReader : public BatchReader {
  public:
+    explicit TestingBatchReader(int32_t* close_count = nullptr) : close_count_(close_count) {}
+
     Result<ReadBatch> NextBatch() override {
         return MakeEofBatch();
     }
@@ -45,7 +47,14 @@ class TestingBatchReader : public BatchReader {
         return nullptr;
     }
 
-    void Close() override {}
+    void Close() override {
+        if (close_count_) {
+            ++(*close_count_);
+        }
+    }
+
+ private:
+    int32_t* close_count_;
 };
 
 TEST(RealtimeReaderTest, TestRejectsIncompleteReader) {
@@ -57,5 +66,21 @@ TEST(RealtimeReaderTest, TestRejectsIncompleteReader) {
                         "inner reader is null");
 }
 
+TEST(RealtimeReaderTest, TestCloseIsIdempotentAndReturnsEof) {
+    int32_t close_count = 0;
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<RealtimeReader> reader,
+        RealtimeReader::Create(std::make_shared<TestingReadView>(),
+                               std::make_unique<TestingBatchReader>(&close_count)));
+    reader->Close();
+    reader->Close();
+    ASSERT_EQ(1, close_count);
+    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch());
+    ASSERT_TRUE(BatchReader::IsEofBatch(batch));
+    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatchWithBitmap batch_with_bitmap,
+                         reader->NextBatchWithBitmap());
+    ASSERT_TRUE(BatchReader::IsEofBatch(batch_with_bitmap));
+}
+
 }  // namespace
 }  // namespace paimon::test
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index dc69ebb55..64d722097 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -91,11 +91,12 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateMemoryReaders(
         if (!reader) {
             return Status::Invalid("PK real-time store returned a null query reader");
         }
-        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> prepared_reader,
-                               AdaptPreparedBatchReader(std::move(reader), prepared_schema,
-                                                        OffsetRange(split->CommittedEndOffset(),
-                                                                    split->MemoryEndOffset()),
-                                                        key_schema, value_schema, memory_pool));
+        PAIMON_ASSIGN_OR_RAISE(
+            std::unique_ptr<KeyValueRecordReader> prepared_reader,
+            AdaptPreparedBatchReader(
+                std::move(reader), prepared_schema,
+                OffsetRange(split->CommittedEndOffset(), split->MemoryEndOffset()), key_schema,
+                value_schema, key_comparator, memory_pool));
         auto merge = std::make_unique<DeduplicateMergeFunction>(false);
         result.push_back(std::make_unique<MergedKeyValueRecordReader>(
             std::move(prepared_reader), key_comparator,
diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp
index dcf10e90c..0bcd79f61 100644
--- a/src/paimon/core/table/source/table_scan.cpp
+++ b/src/paimon/core/table/source/table_scan.cpp
@@ -233,7 +233,7 @@ Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions&
         return Status::Invalid("real-time union read does not support data evolution");
     }
     if (!table_schema.PrimaryKeys().empty()) {
-        PAIMON_RETURN_NOT_OK(ValidatePrimaryKeyRealtimeOptions(core_options));
+        PAIMON_RETURN_NOT_OK(ValidatePrimaryKeyRealtimeOptions(core_options, table_schema));
     }
     if (context.IsStreamingMode()) {
         return Status::Invalid("real-time union read currently supports batch scans only");
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index e68ff670d..e27b36903 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -423,6 +423,208 @@ class SplitCommitReaderRealtimeStoreFactory final : public RealtimeStoreFactory
     ArrowRealtimeStoreFactory delegate_;
 };
 
+class DropLastBatchReader final : public BatchReader {
+ public:
+    explicit DropLastBatchReader(std::unique_ptr<BatchReader> delegate)
+        : delegate_(std::move(delegate)) {}
+
+    Result<ReadBatch> NextBatch() override {
+        if (!buffered_.has_value()) {
+            PAIMON_ASSIGN_OR_RAISE(ReadBatch first, delegate_->NextBatch());
+            if (BatchReader::IsEofBatch(first)) {
+                return MakeEofBatch();
+            }
+            buffered_ = std::move(first);
+        }
+        PAIMON_ASSIGN_OR_RAISE(ReadBatch next, delegate_->NextBatch());
+        if (BatchReader::IsEofBatch(next)) {
+            buffered_.reset();
+            return MakeEofBatch();
+        }
+        ReadBatch result = std::move(buffered_.value());
+        buffered_ = std::move(next);
+        return result;
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return delegate_->GetReaderMetrics();
+    }
+
+    void Close() override {
+        buffered_.reset();
+        delegate_->Close();
+    }
+
+ private:
+    std::unique_ptr<BatchReader> delegate_;
+    std::optional<ReadBatch> buffered_;
+};
+
+class SwapFirstTwoBatchReader final : public BatchReader {
+ public:
+    explicit SwapFirstTwoBatchReader(std::unique_ptr<BatchReader> delegate)
+        : delegate_(std::move(delegate)) {}
+
+    Result<ReadBatch> NextBatch() override {
+        if (!initialized_) {
+            initialized_ = true;
+            PAIMON_ASSIGN_OR_RAISE(ReadBatch first, delegate_->NextBatch());
+            if (BatchReader::IsEofBatch(first)) {
+                return MakeEofBatch();
+            }
+            PAIMON_ASSIGN_OR_RAISE(ReadBatch second, delegate_->NextBatch());
+            if (BatchReader::IsEofBatch(second)) {
+                return first;
+            }
+            first_ = std::move(first);
+            return second;
+        }
+        if (first_.has_value()) {
+            ReadBatch first = std::move(first_.value());
+            first_.reset();
+            return first;
+        }
+        return delegate_->NextBatch();
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return delegate_->GetReaderMetrics();
+    }
+
+    void Close() override {
+        first_.reset();
+        delegate_->Close();
+    }
+
+ private:
+    bool initialized_ = false;
+    std::unique_ptr<BatchReader> delegate_;
+    std::optional<ReadBatch> first_;
+};
+
+class SubstituteOffsetBatchReader final : public BatchReader {
+ public:
+    explicit SubstituteOffsetBatchReader(std::unique_ptr<BatchReader> delegate)
+        : delegate_(std::move(delegate)) {}
+
+    Result<ReadBatch> NextBatch() override {
+        PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, delegate_->NextBatch());
+        if (BatchReader::IsEofBatch(batch)) {
+            return batch;
+        }
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+            std::shared_ptr<arrow::Array> array,
+            arrow::ImportArray(batch.first.get(), batch.second.get()));
+        if (!array || array->type_id() != arrow::Type::STRUCT || array->length() == 0) {
+            return Status::Invalid("offset substitution requires a non-empty struct batch");
+        }
+        std::shared_ptr<arrow::StructArray> struct_array =
+            std::dynamic_pointer_cast<arrow::StructArray>(array);
+        std::shared_ptr<arrow::Int64Array> offsets =
+            std::dynamic_pointer_cast<arrow::Int64Array>(struct_array->field(2));
+        if (!offsets) {
+            return Status::Invalid("offset substitution requires an int64 REALTIME_OFFSET");
+        }
+        arrow::Int64Builder builder;
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Reserve(offsets->length()));
+        for (int64_t row = 0; row < offsets->length(); ++row) {
+            builder.UnsafeAppend(0);
+        }
+        std::shared_ptr<arrow::Array> substituted_offsets;
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&substituted_offsets));
+        std::shared_ptr<arrow::ArrayData> substituted_data = struct_array->data()->Copy();
+        substituted_data->child_data[2] = substituted_offsets->data();
+        std::shared_ptr<arrow::Array> substituted = arrow::MakeArray(std::move(substituted_data));
+        auto output = std::make_unique<ArrowArray>();
+        auto schema = std::make_unique<ArrowSchema>();
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(
+            arrow::ExportArray(*substituted, output.get(), schema.get()));
+        return ReadBatch(std::move(output), std::move(schema));
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return delegate_->GetReaderMetrics();
+    }
+
+    void Close() override {
+        delegate_->Close();
+    }
+
+ private:
+    std::unique_ptr<BatchReader> delegate_;
+};
+
+enum class CommitReaderMalformation { DROP_LAST, UNSORTED, SUBSTITUTE_OFFSET };
+
+class MalformedCoverageRealtimeStore final : public RealtimeStore {
+ public:
+    MalformedCoverageRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
+                                   CommitReaderMalformation malformation)
+        : delegate_(delegate), malformation_(malformation) {}
+
+    Status Write(RealtimeWriteBatch&& batch) override {
+        return delegate_->Write(std::move(batch));
+    }
+    Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>> SealForCommit() override {
+        return delegate_->SealForCommit();
+    }
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
+        const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
+        PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
+                               delegate_->CreateCommitReaders(segment));
+        for (std::unique_ptr<BatchReader>& reader : readers) {
+            switch (malformation_) {
+                case CommitReaderMalformation::DROP_LAST:
+                    reader = std::make_unique<DropLastBatchReader>(std::move(reader));
+                    break;
+                case CommitReaderMalformation::UNSORTED:
+                    reader = std::make_unique<SwapFirstTwoBatchReader>(std::move(reader));
+                    break;
+                case CommitReaderMalformation::SUBSTITUTE_OFFSET:
+                    reader = std::make_unique<SubstituteOffsetBatchReader>(std::move(reader));
+                    break;
+            }
+        }
+        return readers;
+    }
+    Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
+        return delegate_->AcquireReadView();
+    }
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
+        const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
+        const RealtimeQueryContext& context) override {
+        return delegate_->CreateQueryReaders(view, offset_begin, context);
+    }
+    Status AdvanceCommittedOffset(int64_t committed_offset) override {
+        return delegate_->AdvanceCommittedOffset(committed_offset);
+    }
+    uint64_t GetMemoryUsage() const override {
+        return delegate_->GetMemoryUsage();
+    }
+
+ private:
+    std::shared_ptr<RealtimeStore> delegate_;
+    CommitReaderMalformation malformation_;
+};
+
+class MalformedCoverageRealtimeStoreFactory final : public RealtimeStoreFactory {
+ public:
+    explicit MalformedCoverageRealtimeStoreFactory(
+        CommitReaderMalformation malformation = CommitReaderMalformation::DROP_LAST)
+        : malformation_(malformation) {}
+
+    Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> delegate,
+                               delegate_.Create(std::move(request)));
+        return std::shared_ptr<RealtimeStore>(
+            std::make_shared<MalformedCoverageRealtimeStore>(delegate, malformation_));
+    }
+
+ private:
+    ArrowRealtimeStoreFactory delegate_;
+    CommitReaderMalformation malformation_;
+};
+
 }  // namespace
 
 namespace {
@@ -1312,6 +1514,50 @@ TEST_F(RealtimeWriteInteTest, TestPkRead) {
     ASSERT_TRUE(query_view->expired());
 }
 
+TEST_F(RealtimeWriteInteTest, TestPkDeleteInsertAndPinnedReadsAcrossRefresh) {
+    CreatePkTable();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> delete_batch,
+                         MakeBatch({Row{1, "deleted", "p0"}}, /*partitioned=*/false, /*bucket=*/0,
+                                   {RecordBatch::RowKind::DELETE}));
+    ASSERT_OK(writer->Write(std::move(delete_batch)));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> insert_batch,
+                         MakeBatch({Row{1, "inserted", "p0"}}, /*partitioned=*/false, /*bucket=*/0,
+                                   {RecordBatch::RowKind::INSERT}));
+    ASSERT_OK(writer->Write(std::move(insert_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_EQ(1, progress.size());
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> pinned_plan,
+                         CreatePlan(realtime_context, /*predicate=*/nullptr));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> reader_plan,
+                         CreatePlan(realtime_context, /*predicate=*/nullptr));
+    ReadContextBuilder read_builder(table_path_);
+    read_builder.SetOptions(options_)
+        .SetReadFieldNames({"id", "payload", "pt"})
+        .WithRealtimeContext(realtime_context)
+        .WithMemoryPool(pool_);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableRead> table_read,
+                         TableRead::Create(std::move(read_context)));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> pinned_reader,
+                         table_read->CreateReader(reader_plan->Splits()));
+
+    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, /*commit_identifier=*/0));
+    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> plan_rows, ReadRows(pinned_plan, realtime_context));
+    ASSERT_EQ((std::vector<Row>{{1, "inserted", "p0"}}), plan_rows);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> reader_rows,
+                         ReadResultCollector::CollectResult(pinned_reader.get()));
+    ASSERT_EQ(1, reader_rows->length());
+    ASSERT_OK(writer->Close());
+}
+
 TEST_F(RealtimeWriteInteTest, TestPkMergeDiskSealedAndActive) {
     options_[Options::READ_BATCH_SIZE] = "2";
     CreatePkTable();
@@ -1903,6 +2149,56 @@ TEST_F(RealtimeWriteInteTest, TestPkPluginContract) {
     ASSERT_OK(writer->Close());
 }
 
+TEST_F(RealtimeWriteInteTest, TestPkRejectsMalformedCoverage) {
+    CreatePkTable();
+    auto factory = std::make_shared<MalformedCoverageRealtimeStoreFactory>();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create(factory));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         MakeBatch({Row{1, "one", "p0"}, Row{2, "two", "p0"}},
+                                   /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(batch)));
+    ASSERT_NOK_WITH_MSG(writer->PrepareCommitWithProgress(/*commit_identifier=*/0),
+                        "commit readers did not cover the sealed range");
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestPkRejectsEqualCardinalityOffsetSubstitution) {
+    CreatePkTable();
+    auto factory = std::make_shared<MalformedCoverageRealtimeStoreFactory>(
+        CommitReaderMalformation::SUBSTITUTE_OFFSET);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create(factory));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         MakeBatch({Row{1, "one", "p0"}, Row{2, "two", "p0"}},
+                                   /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(batch)));
+    ASSERT_NOK_WITH_MSG(writer->PrepareCommitWithProgress(/*commit_identifier=*/0),
+                        "duplicate REALTIME_OFFSET");
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestPkRejectsUnsortedPluginRows) {
+    CreatePkTable();
+    auto factory =
+        std::make_shared<MalformedCoverageRealtimeStoreFactory>(CommitReaderMalformation::UNSORTED);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create(factory));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         MakeBatch({Row{1, "one", "p0"}, Row{2, "two", "p0"}},
+                                   /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(batch)));
+    ASSERT_NOK_WITH_MSG(writer->PrepareCommitWithProgress(/*commit_identifier=*/0),
+                        "not globally sorted by primary key and sequence number");
+    ASSERT_OK(writer->Close());
+}
+
 TEST_F(RealtimeWriteInteTest, TestPkQueryReaderClose) {
     CreatePkTable();
     auto state = std::make_shared<CloseTrackingReaderState>();
@@ -1973,10 +2269,10 @@ TEST_F(RealtimeWriteInteTest, TestPkQueryReaderCloseFailure) {
         return table_read->CreateReader(plan->Splits());
     };
 
-    for (int32_t null_index = 0; null_index <= 2; ++null_index) {
+    for (int32_t null_index = 0; null_index <= 1; ++null_index) {
         state->query_null_index = null_index;
         ASSERT_NOK_WITH_MSG(create_reader(), "PK real-time store returned a null query reader");
-        ASSERT_EQ(2 * (null_index + 1), state->query_close_count->load(std::memory_order_acquire));
+        ASSERT_EQ(null_index + 1, state->query_close_count->load(std::memory_order_acquire));
     }
     ASSERT_OK(writer->Close());
 }

From 1e1c7d2660dc0c21d42a8bda0aecca64c2f8868a Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Mon, 24 Aug 2026 18:27:48 +0800
Subject: [PATCH 21/62] refactor(realtime): simplify reader lifecycle cleanup

---
 .../merged_key_value_record_reader_test.cpp   |  9 +------
 .../key_value_file_store_write_test.cpp       | 10 ++++---
 .../realtime/arrow_realtime_store_test.cpp    |  8 +++++-
 .../realtime/prepared_key_value_reader.cpp    |  4 ---
 .../realtime/primary_key_realtime_store.cpp   |  8 ------
 .../primary_key_realtime_store_test.cpp       |  3 ---
 .../realtime/realtime_append_only_writer.cpp  |  2 +-
 .../core/realtime/realtime_context_impl.cpp   | 26 +++++++++++++++----
 .../core/realtime/realtime_context_impl.h     |  3 +++
 src/paimon/core/realtime/realtime_reader.h    | 11 --------
 .../core/realtime/realtime_reader_test.cpp    | 15 +++++------
 test/inte/realtime_write_inte_test.cpp        |  2 +-
 12 files changed, 48 insertions(+), 53 deletions(-)

diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index 775f271e3..a83c9bbd6 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -81,16 +81,11 @@ class TrackingBatchReader : public BatchReader {
     }
 
     void Close() override {
-        if (closed_) {
-            return;
-        }
-        closed_ = true;
         ++(*close_count_);
         delegate_->Close();
     }
 
  private:
-    bool closed_ = false;
     std::unique_ptr<BatchReader> delegate_;
     int32_t* close_count_;
 };
@@ -421,7 +416,7 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderNestedProjection) {
     ASSERT_EQ(keyed_value_map->ValueArray()->GetInt(1), 23);
 }
 
-TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderClose) {
+TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderLifecycle) {
     std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32())),
                                            DataField(1, arrow::field("v0", arrow::int32()))};
     std::shared_ptr<arrow::Schema> value_schema =
@@ -445,7 +440,6 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderClose) {
             AdaptPreparedBatchReader(std::move(tracking_reader), prepared_schema, OffsetRange(0, 1),
                                      key_schema, value_schema, pool_));
         reader->Close();
-        reader->Close();
     }
     ASSERT_EQ(explicit_close_count, 1);
 
@@ -486,7 +480,6 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderClose) {
                                      key_schema, value_schema, pool_));
         ASSERT_NOK_WITH_MSG(reader->NextBatch(), "prepared reader failure");
         ASSERT_EQ(read_failure_close_count, 1);
-        reader->Close();
     }
     ASSERT_EQ(read_failure_close_count, 1);
 }
diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp
index 733c19d6e..a2344e803 100644
--- a/src/paimon/core/operation/key_value_file_store_write_test.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp
@@ -411,6 +411,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeWrite) {
     const std::map<std::string, std::string> options = {
         {Options::BUCKET, "1"},
         {Options::WRITE_BUFFER_SIZE, "1"},
+        {Options::REALTIME_ENABLED, "true"},
     };
     const std::shared_ptr<arrow::Schema> schema = arrow::schema({
         arrow::field("id", arrow::int64(), false),
@@ -465,7 +466,8 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeWrite) {
 }
 
 TEST_F(KeyValueFileStoreWriteTest, TestRealtimeOffsetCollision) {
-    const std::map<std::string, std::string> options = {{Options::BUCKET, "1"}};
+    const std::map<std::string, std::string> options = {
+        {Options::BUCKET, "1"}, {Options::REALTIME_ENABLED, "true"}};
     const std::shared_ptr<arrow::Schema> schema = arrow::schema({
         arrow::field("id", arrow::int64(), false),
         arrow::field("_REALTIME_OFFSET", arrow::int64()),
@@ -488,7 +490,8 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeOffsetCollision) {
 }
 
 TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) {
-    const std::map<std::string, std::string> options = {{Options::BUCKET, "1"}};
+    const std::map<std::string, std::string> options = {
+        {Options::BUCKET, "1"}, {Options::REALTIME_ENABLED, "true"}};
     const std::shared_ptr<arrow::Schema> schema = arrow::schema({
         arrow::field("id", arrow::int64(), false),
         arrow::field("value", arrow::utf8()),
@@ -546,7 +549,8 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) {
 
 TEST_F(KeyValueFileStoreWriteTest, TestRealtimeLimits) {
     const int64_t max = std::numeric_limits<int64_t>::max();
-    const std::map<std::string, std::string> options = {{Options::BUCKET, "1"}};
+    const std::map<std::string, std::string> options = {
+        {Options::BUCKET, "1"}, {Options::REALTIME_ENABLED, "true"}};
     const std::shared_ptr<arrow::Schema> schema = arrow::schema({
         arrow::field("id", arrow::int64(), false),
         arrow::field("value", arrow::utf8()),
diff --git a/src/paimon/core/realtime/arrow_realtime_store_test.cpp b/src/paimon/core/realtime/arrow_realtime_store_test.cpp
index 9aae99332..f186a8161 100644
--- a/src/paimon/core/realtime/arrow_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/arrow_realtime_store_test.cpp
@@ -232,8 +232,14 @@ TEST_F(ArrowRealtimeStoreTest, TestCommitReaderPreservesSlicedBatch) {
 TEST_F(ArrowRealtimeStoreTest, TestFullStatisticsPrunesNonMatchingBatch) {
     ArrowRealtimeStoreFactory factory;
     std::unique_ptr<ArrowSchema> write_schema = MakeReadSchema(schema_);
+    RealtimeStoreCreateRequest request{std::move(write_schema),
+                                       /*options=*/{},
+                                       pool_,
+                                       /*partition=*/{},
+                                       /*bucket=*/0,
+                                       AppendRealtimeStoreCreateConfig{StatisticsMode::FULL}};
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeStore> realtime_store,
-                         factory.Create(std::move(write_schema), StatisticsMode::FULL, {}, pool_));
+                         factory.Create(std::move(request)));
     std::shared_ptr<ArrowRealtimeStore> store =
         std::dynamic_pointer_cast<ArrowRealtimeStore>(realtime_store);
     ASSERT_NE(nullptr, store);
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index 6b3afcd19..5b0375ad1 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -490,10 +490,6 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
 
  private:
     Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatchImpl() {
-        if (closed_) {
-            return std::unique_ptr<KeyValueRecordReader::Iterator>();
-        }
-
         while (true) {
             ResetBatchState();
             PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader_->NextBatch());
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index f43f60472..2f04aae79 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -177,9 +177,6 @@ class RawBatchReader final : public BatchReader {
     }
 
     Result<ReadBatch> NextBatch() override {
-        if (closed_) {
-            return MakeEofBatch();
-        }
         std::optional<size_t> selected;
         for (size_t i = 0; i < batches_.size(); ++i) {
             if (positions_[i] >= batches_[i].data->length()) {
@@ -214,10 +211,6 @@ class RawBatchReader final : public BatchReader {
         return metrics_;
     }
     void Close() override {
-        if (closed_) {
-            return;
-        }
-        closed_ = true;
         batches_.clear();
         positions_.clear();
         key_contexts_.clear();
@@ -238,7 +231,6 @@ class RawBatchReader final : public BatchReader {
         return left_sequences->Value(positions_[left]) < right_sequences->Value(positions_[right]);
     }
 
-    bool closed_ = false;
     std::vector<StoredBatch> batches_;
     std::vector<int64_t> positions_;
     std::vector<int32_t> key_field_indexes_;
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index cafe3682e..116c6e389 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -218,9 +218,6 @@ TEST(PrimaryKeyRealtimeStoreTest, TestCommitBatches) {
         "4 type: string\n  [\n    \"before\",\n    \"after\",\n    \"three\"\n  ]",
         actual);
     readers[0]->Close();
-    readers[0]->Close();
-    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch eof, readers[0]->NextBatch());
-    ASSERT_TRUE(BatchReader::IsEofBatch(eof));
 }
 
 TEST(PrimaryKeyRealtimeStoreTest, TestCommitReaderExportsZeroOffsets) {
diff --git a/src/paimon/core/realtime/realtime_append_only_writer.cpp b/src/paimon/core/realtime/realtime_append_only_writer.cpp
index 21d6cfb74..ea5feecce 100644
--- a/src/paimon/core/realtime/realtime_append_only_writer.cpp
+++ b/src/paimon/core/realtime/realtime_append_only_writer.cpp
@@ -47,7 +47,7 @@ Result<std::shared_ptr<RealtimeAppendOnlyWriter>> RealtimeAppendOnlyWriter::Crea
     std::unique_ptr<::ArrowSchema> write_schema,
     const std::shared_ptr<RealtimeContext>& realtime_context,
     const std::shared_ptr<AppendOnlyWriter>& file_writer,
-    const std::shared_ptr<arrow::Schema>& input_schema,
+    const std::shared_ptr<arrow::Schema>& input_schema, StatisticsMode statistics_mode,
     const std::map<std::string, std::string>& options,
     const std::shared_ptr<MemoryPool>& memory_pool) {
     if (!realtime_context) {
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index 215e066ee..ba4c8b7a6 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -269,12 +269,28 @@ Status RealtimeContextImpl::AdvanceCommittedProgress(int64_t snapshot_id,
             if (partition_bucket.bucket < 0 || committed_end_offset < 0) {
                 return Status::Invalid("invalid partition-bucket committed offset");
             }
+        }
+        // Only stores created by this context can contain state which cannot be restored in
+        // place. Offsets for other partition-buckets are reference state for lazy store creation
+        // and may be removed or rolled back without rebuilding the context.
+        std::lock_guard<std::mutex> registry_lock(mutex_);
+        for (const auto& store_entry : stores_) {
+            const RealtimePartitionBucket& partition_bucket = store_entry.first;
             auto previous_iter = committed_offsets_.find(partition_bucket);
-            if (previous_iter != committed_offsets_.end()) {
-                if (committed_end_offset < previous_iter->second) {
-                    return Status::Invalid(
-                        "real-time partition-bucket committed offset cannot move backwards");
-                }
+            if (previous_iter == committed_offsets_.end()) {
+                continue;
+            }
+
+            auto current_iter = committed_offsets.find(partition_bucket);
+            if (current_iter == committed_offsets.end()) {
+                return Status::Invalid(
+                    "real-time committed progress removed an active partition-bucket; recreate "
+                    "RealtimeContext");
+            }
+            if (current_iter->second < previous_iter->second) {
+                return Status::Invalid(
+                    "real-time committed offset moved backwards for an active partition-bucket; "
+                    "recreate RealtimeContext");
             }
         }
         committed_offsets_ = committed_offsets;
diff --git a/src/paimon/core/realtime/realtime_context_impl.h b/src/paimon/core/realtime/realtime_context_impl.h
index 9fa145e99..f5118c18f 100644
--- a/src/paimon/core/realtime/realtime_context_impl.h
+++ b/src/paimon/core/realtime/realtime_context_impl.h
@@ -88,6 +88,9 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
 
     Status ReleaseReadView(const std::string& opaque_ticket);
 
+    // Returns an error requiring a new context if a newer snapshot removes or moves committed
+    // progress backwards for a store created by this context. Progress for inactive stores is
+    // only reference state and can be replaced in place.
     Status AdvanceCommittedProgress(int64_t snapshot_id,
                                     const RealtimeOffsetMap& committed_offsets);
 
diff --git a/src/paimon/core/realtime/realtime_reader.h b/src/paimon/core/realtime/realtime_reader.h
index a041e0caa..6c25fd853 100644
--- a/src/paimon/core/realtime/realtime_reader.h
+++ b/src/paimon/core/realtime/realtime_reader.h
@@ -44,16 +44,10 @@ class RealtimeReader final : public BatchReader {
     }
 
     Result<ReadBatch> NextBatch() override {
-        if (closed_) {
-            return MakeEofBatch();
-        }
         return reader_->NextBatch();
     }
 
     Result<ReadBatchWithBitmap> NextBatchWithBitmap() override {
-        if (closed_) {
-            return MakeEofBatchWithBitmap();
-        }
         return reader_->NextBatchWithBitmap();
     }
 
@@ -62,10 +56,6 @@ class RealtimeReader final : public BatchReader {
     }
 
     void Close() override {
-        if (closed_) {
-            return;
-        }
-        closed_ = true;
         reader_->Close();
         read_view_.reset();
     }
@@ -78,7 +68,6 @@ class RealtimeReader final : public BatchReader {
     // before releasing the data it references.
     std::shared_ptr<RealtimeReadView> read_view_;
     std::unique_ptr<BatchReader> reader_;
-    bool closed_ = false;
 };
 
 }  // namespace paimon
diff --git a/src/paimon/core/realtime/realtime_reader_test.cpp b/src/paimon/core/realtime/realtime_reader_test.cpp
index 10f6ce5be..ded060989 100644
--- a/src/paimon/core/realtime/realtime_reader_test.cpp
+++ b/src/paimon/core/realtime/realtime_reader_test.cpp
@@ -21,6 +21,7 @@
 
 #include <memory>
 #include <optional>
+#include <utility>
 
 #include "paimon/arrow/abi.h"
 #include "paimon/testing/utils/testharness.h"
@@ -66,20 +67,18 @@ TEST(RealtimeReaderTest, TestRejectsIncompleteReader) {
                         "inner reader is null");
 }
 
-TEST(RealtimeReaderTest, TestCloseIsIdempotentAndReturnsEof) {
+TEST(RealtimeReaderTest, TestCloseReleasesResources) {
     int32_t close_count = 0;
+    std::shared_ptr<TestingReadView> read_view = std::make_shared<TestingReadView>();
+    std::weak_ptr<TestingReadView> weak_read_view = read_view;
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<RealtimeReader> reader,
-        RealtimeReader::Create(std::make_shared<TestingReadView>(),
+        RealtimeReader::Create(std::move(read_view),
                                std::make_unique<TestingBatchReader>(&close_count)));
-    reader->Close();
+    ASSERT_FALSE(weak_read_view.expired());
     reader->Close();
     ASSERT_EQ(1, close_count);
-    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch());
-    ASSERT_TRUE(BatchReader::IsEofBatch(batch));
-    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatchWithBitmap batch_with_bitmap,
-                         reader->NextBatchWithBitmap());
-    ASSERT_TRUE(BatchReader::IsEofBatch(batch_with_bitmap));
+    ASSERT_TRUE(weak_read_view.expired());
 }
 
 }  // namespace
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index e27b36903..0e6f83b70 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -1939,7 +1939,7 @@ TEST_F(RealtimeWriteInteTest, TestPkRecovery) {
                          seed_commit_builder.SetOptions(options_).Finish());
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> seed_commit,
                          FileStoreCommit::Create(std::move(seed_commit_context)));
-    ASSERT_OK(seed_commit->Commit(seed_messages));
+    ASSERT_OK(seed_commit->Commit(seed_messages, /*commit_identifier=*/0));
     ASSERT_OK(seed_writer->Close());
     const std::vector<Row> mutations = {
         {1, "one", "p0"}, {1, "one-new", "p0"}, {2, "deleted", "p0"}, {3, "three", "p0"}};

From 352ce215288ee17b8f0a99d84a2e6452471a7066 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Mon, 24 Aug 2026 23:01:26 +0800
Subject: [PATCH 22/62] fix(realtime): harden primary-key prepared batches

---
 include/paimon/realtime/realtime_store.h      |   2 +
 include/paimon/utils/special_field_ids.h      |   2 +
 .../io/merged_key_value_record_reader.cpp     |  10 +-
 .../core/io/merged_key_value_record_reader.h  |   1 +
 .../merged_key_value_record_reader_test.cpp   |  93 ++++++++-
 src/paimon/core/mergetree/merge_tree_writer.h |   3 +
 .../realtime/prepared_key_value_reader.cpp    | 144 ++++++++------
 .../core/realtime/prepared_key_value_reader.h |   2 +
 .../realtime/primary_key_realtime_store.cpp   | 135 ++++++++++---
 .../primary_key_realtime_store_test.cpp       | 128 ++++++++++++-
 src/paimon/core/realtime/realtime_fields.h    |   6 +-
 .../realtime/realtime_primary_key_writer.cpp  |  14 --
 .../table/source/append_only_table_read.cpp   |  37 +++-
 .../table/source/key_value_table_read.cpp     |  12 +-
 .../core/table/source/realtime_table_scan.cpp |  20 +-
 .../core/table/source/realtime_table_scan.h   |   3 +-
 src/paimon/core/table/source/table_scan.cpp   |   2 +-
 test/inte/realtime_write_inte_test.cpp        | 180 ++++++++++++++++--
 18 files changed, 652 insertions(+), 142 deletions(-)

diff --git a/include/paimon/realtime/realtime_store.h b/include/paimon/realtime/realtime_store.h
index 792bb1c56..90c6ce0a8 100644
--- a/include/paimon/realtime/realtime_store.h
+++ b/include/paimon/realtime/realtime_store.h
@@ -109,6 +109,8 @@ class PAIMON_EXPORT RealtimeReadView {
 struct PAIMON_EXPORT RealtimeQueryContext {
     /// Append mode receives the requested output fields before the mandatory leading
     /// `_VALUE_KIND` field is added. Primary-key mode receives the complete prepared schema.
+    /// This schema is borrowed and remains valid only during `CreateQueryReaders`; plugins must
+    /// import or copy it synchronously.
     ::ArrowSchema* read_schema;
     /// Predicate using field indexes from `read_schema`.
     std::shared_ptr<Predicate> predicate;
diff --git a/include/paimon/utils/special_field_ids.h b/include/paimon/utils/special_field_ids.h
index 829f29889..5219d72db 100644
--- a/include/paimon/utils/special_field_ids.h
+++ b/include/paimon/utils/special_field_ids.h
@@ -42,6 +42,8 @@ class SpecialFieldIds {
 
     /// Special field ID reserved for index score. Value: CPP_FIELD_ID_END - 1
     inline static constexpr int32_t INDEX_SCORE = CPP_FIELD_ID_END - 1;
+    /// Special field ID reserved for realtime offset. Value: CPP_FIELD_ID_END - 2
+    inline static constexpr int32_t REALTIME_OFFSET = CPP_FIELD_ID_END - 2;
 
     /// Lowest field ID reserved for system fields; IDs at or above it are excluded from the
     /// highest field ID of a schema. Value: INT32_MAX / 2
diff --git a/src/paimon/core/io/merged_key_value_record_reader.cpp b/src/paimon/core/io/merged_key_value_record_reader.cpp
index 70f2bcfb9..8c3952874 100644
--- a/src/paimon/core/io/merged_key_value_record_reader.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader.cpp
@@ -117,13 +117,21 @@ Status MergedKeyValueRecordReader::Iterator::LoadNextRawKeyValue() const {
 }
 
 Result<std::unique_ptr<KeyValueRecordReader::Iterator>> MergedKeyValueRecordReader::NextBatch() {
+    if (initialization_error_.has_value()) {
+        return initialization_error_.value();
+    }
     if (visited_) {
         return std::unique_ptr<KeyValueRecordReader::Iterator>();
     }
     visited_ = true;
 
     auto iterator = std::make_unique<Iterator>(this);
-    PAIMON_ASSIGN_OR_RAISE(bool has_next, iterator->HasNext());
+    Result<bool> has_next_result = iterator->HasNext();
+    if (!has_next_result.ok()) {
+        initialization_error_ = has_next_result.status();
+        return initialization_error_.value();
+    }
+    bool has_next = std::move(has_next_result).value();
     if (!has_next) {
         return std::unique_ptr<KeyValueRecordReader::Iterator>();
     }
diff --git a/src/paimon/core/io/merged_key_value_record_reader.h b/src/paimon/core/io/merged_key_value_record_reader.h
index a1b7aa5e4..227a1593a 100644
--- a/src/paimon/core/io/merged_key_value_record_reader.h
+++ b/src/paimon/core/io/merged_key_value_record_reader.h
@@ -67,6 +67,7 @@ class MergedKeyValueRecordReader : public KeyValueRecordReader {
 
  private:
     bool visited_ = false;
+    std::optional<Status> initialization_error_;
     std::unique_ptr<KeyValueRecordReader> reader_;
     std::shared_ptr<FieldsComparator> key_comparator_;
     std::shared_ptr<MergeFunctionWrapper<KeyValue>> merge_function_wrapper_;
diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index a83c9bbd6..a0d65205c 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -19,7 +19,6 @@
 #include "paimon/core/io/merged_key_value_record_reader.h"
 
 #include <cstdint>
-#include <limits>
 #include <memory>
 #include <optional>
 #include <string>
@@ -45,6 +44,7 @@
 #include "paimon/testing/utils/key_value_checker.h"
 #include "paimon/testing/utils/read_result_collector.h"
 #include "paimon/testing/utils/testharness.h"
+#include "paimon/utils/special_field_ids.h"
 
 namespace paimon::test {
 
@@ -107,7 +107,7 @@ class MergedKeyValueRecordReaderTest : public testing::Test {
 
 TEST_F(MergedKeyValueRecordReaderTest, TestRealtimeOffsetField) {
     const DataField& field = RealtimeOffsetField();
-    ASSERT_EQ(std::numeric_limits<int32_t>::max() - 10002, field.Id());
+    ASSERT_EQ(SpecialFieldIds::REALTIME_OFFSET, field.Id());
     ASSERT_EQ("_REALTIME_OFFSET", field.Name());
     ASSERT_EQ(arrow::Type::INT64, field.Type()->id());
     ASSERT_FALSE(field.Nullable());
@@ -296,6 +296,95 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderCommitSchema) {
                         "exact");
 }
 
+TEST_F(MergedKeyValueRecordReaderTest, TestBadCommitBatch) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Field> value = MakeField("value", arrow::int32(), 1);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key, value});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key, value});
+    std::shared_ptr<arrow::Schema> actual_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> actual_type = arrow::struct_(actual_schema->fields());
+    std::shared_ptr<arrow::Array> actual =
+        arrow::ipc::internal::json::ArrayFromJSON(actual_type, R"([[0, 10, 0, 1]])").ValueOrDie();
+
+    auto batch_reader = std::make_unique<MockFileBatchReader>(actual, actual_type, 1);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, std::nullopt,
+                                 arrow::schema({key}), value_schema, pool_));
+    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "field count");
+}
+
+TEST_F(MergedKeyValueRecordReaderTest, TestMissingCompositeKey) {
+    std::shared_ptr<arrow::Field> key0 = MakeField("key0", arrow::int32(), 0);
+    std::shared_ptr<arrow::Field> key1 = MakeField("key1", arrow::int32(), 1);
+    std::shared_ptr<arrow::Field> value = MakeField("value", arrow::int32(), 2);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key0, key1, value});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key0, key1, value});
+    std::shared_ptr<arrow::Schema> actual_schema = MakePreparedSchema({key0, value});
+    std::shared_ptr<arrow::DataType> actual_type = arrow::struct_(actual_schema->fields());
+    std::shared_ptr<arrow::Array> actual =
+        arrow::ipc::internal::json::ArrayFromJSON(actual_type, R"([[0, 10, 0, 1, 20]])")
+            .ValueOrDie();
+
+    auto batch_reader = std::make_unique<MockFileBatchReader>(actual, actual_type, 1);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
+                                 arrow::schema({key0, key1}), value_schema, pool_));
+    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "cannot find field id 1");
+}
+
+TEST_F(MergedKeyValueRecordReaderTest, TestQueryAddRename) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Field> old_value = MakeField("old_value", arrow::int32(), 1);
+    std::shared_ptr<arrow::Field> renamed_value = MakeField("renamed_value", arrow::int32(), 1);
+    std::shared_ptr<arrow::Field> added = MakeField("added", arrow::int32(), 2);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key, renamed_value, added});
+    std::shared_ptr<arrow::Schema> prepared_schema =
+        MakePreparedSchema({key, renamed_value, added});
+    std::shared_ptr<arrow::Schema> actual_schema = MakePreparedSchema({key, old_value});
+    std::shared_ptr<arrow::DataType> actual_type = arrow::struct_(actual_schema->fields());
+    std::shared_ptr<arrow::Array> actual =
+        arrow::ipc::internal::json::ArrayFromJSON(actual_type, R"([[0, 10, 0, 1, 20]])")
+            .ValueOrDie();
+
+    auto batch_reader = std::make_unique<MockFileBatchReader>(actual, actual_type, 1);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
+                                 arrow::schema({key}), value_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader::Iterator> iterator,
+                         reader->NextBatch());
+    ASSERT_OK_AND_ASSIGN(KeyValue key_value, iterator->Next());
+    ASSERT_EQ(20, key_value.value->GetInt(1));
+    ASSERT_TRUE(key_value.value->IsNullAt(2));
+}
+
+TEST_F(MergedKeyValueRecordReaderTest, TestMergedReaderErrorRetry) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1]])").ValueOrDie();
+    auto failing_reader = std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
+    failing_reader->SetNextBatchStatus(Status::IOError("stable prepared error"));
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        AdaptPreparedBatchReader(std::move(failing_reader), prepared_schema, OffsetRange(0, 1),
+                                 value_schema, value_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<FieldsComparator> key_comparator,
+                         FieldsComparator::Create({DataField(0, key)}, true));
+    MergedKeyValueRecordReader merged_reader(std::move(reader), key_comparator,
+                                             merge_function_wrapper_);
+
+    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> first = merged_reader.NextBatch();
+    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> retry = merged_reader.NextBatch();
+    ASSERT_NOK(first);
+    ASSERT_NOK(retry);
+    ASSERT_EQ(first.status().ToString(), retry.status().ToString());
+}
+
 TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderSafeDecode) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
diff --git a/src/paimon/core/mergetree/merge_tree_writer.h b/src/paimon/core/mergetree/merge_tree_writer.h
index 542affd81..cea07f3e4 100644
--- a/src/paimon/core/mergetree/merge_tree_writer.h
+++ b/src/paimon/core/mergetree/merge_tree_writer.h
@@ -70,6 +70,9 @@ class MergeTreeWriter : public BatchWriter {
 
     Status Write(std::unique_ptr<RecordBatch>&& batch) override;
 
+    /// Consumes readers whose complete streams are individually sorted by primary key and
+    /// sequence number. Readers are closed on success or failure; an error may leave generated
+    /// file state unpublished, so the caller must discard this writer and replay its input.
     Status WriteSortedReaders(std::vector<std::unique_ptr<KeyValueRecordReader>>&& readers);
 
     Status Compact(bool full_compaction) override;
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index 5b0375ad1..864456818 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -62,8 +62,18 @@ constexpr int32_t kSequenceNumberIndex = 1;
 constexpr int32_t kRealtimeOffsetIndex = 2;
 constexpr int32_t kPreparedValueStartIndex = 3;
 
+template <typename Reader>
+void CloseReaders(const std::vector<std::unique_ptr<Reader>>& readers) {
+    for (const std::unique_ptr<Reader>& reader : readers) {
+        if (reader) {
+            reader->Close();
+        }
+    }
+}
+
 Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
-    const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& read_type);
+    const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& read_type,
+    arrow::MemoryPool* arrow_pool);
 
 class RealtimeOffsetCoverage {
  public:
@@ -237,22 +247,9 @@ Status ValidateExactCommitSchema(const std::shared_ptr<arrow::Schema>& prepared_
     return Status::OK();
 }
 
-Status ValidatePreparedSchema(const std::shared_ptr<arrow::Schema>& prepared_schema) {
-    if (!prepared_schema || prepared_schema->num_fields() < kPreparedValueStartIndex) {
-        return Status::Invalid("prepared schema must contain realtime transport fields");
-    }
-    PAIMON_RETURN_NOT_OK(
-        CheckPreparedField(prepared_schema, kValueKindIndex, SpecialFields::ValueKind()));
-    PAIMON_RETURN_NOT_OK(
-        CheckPreparedField(prepared_schema, kSequenceNumberIndex, SpecialFields::SequenceNumber()));
-    PAIMON_RETURN_NOT_OK(
-        CheckPreparedField(prepared_schema, kRealtimeOffsetIndex, RealtimeOffsetField()));
-    return Status::OK();
-}
-
 Result<std::shared_ptr<arrow::Array>> AlignStructArrayByPaimonIds(
     const std::shared_ptr<arrow::StructArray>& array,
-    const std::shared_ptr<arrow::StructType>& read_type) {
+    const std::shared_ptr<arrow::StructType>& read_type, arrow::MemoryPool* arrow_pool) {
     const std::shared_ptr<arrow::StructType> data_type =
         checked_pointer_cast<arrow::StructType>(array->type());
     std::unordered_map<int32_t, int32_t> data_field_id_to_idx;
@@ -273,12 +270,16 @@ Result<std::shared_ptr<arrow::Array>> AlignStructArrayByPaimonIds(
                                NestedProjectionUtils::GetPaimonFieldId(read_field));
         auto data_iter = data_field_id_to_idx.find(read_field_id);
         if (data_iter == data_field_id_to_idx.end()) {
-            return Status::Invalid(
-                fmt::format("cannot find field id {} in prepared value struct", read_field_id));
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+                std::shared_ptr<arrow::Array> null_child,
+                arrow::MakeArrayOfNull(read_field->type(), array->offset() + array->length(),
+                                       arrow_pool));
+            aligned_arrays.push_back(std::move(null_child));
+            continue;
         }
         std::shared_ptr<arrow::Array> child =
             arrow::MakeArray(array->data()->child_data[data_iter->second]);
-        PAIMON_ASSIGN_OR_RAISE(child, AlignArrayByPaimonIds(child, read_field->type()));
+        PAIMON_ASSIGN_OR_RAISE(child, AlignArrayByPaimonIds(child, read_field->type(), arrow_pool));
         aligned_arrays.push_back(std::move(child));
     }
 
@@ -294,9 +295,10 @@ Result<std::shared_ptr<arrow::Array>> AlignStructArrayByPaimonIds(
 
 Result<std::shared_ptr<arrow::Array>> AlignListArrayByPaimonIds(
     const std::shared_ptr<arrow::ListArray>& array,
-    const std::shared_ptr<arrow::ListType>& read_type) {
+    const std::shared_ptr<arrow::ListType>& read_type, arrow::MemoryPool* arrow_pool) {
     std::shared_ptr<arrow::Array> values = array->values();
-    PAIMON_ASSIGN_OR_RAISE(values, AlignArrayByPaimonIds(values, read_type->value_type()));
+    PAIMON_ASSIGN_OR_RAISE(values,
+                           AlignArrayByPaimonIds(values, read_type->value_type(), arrow_pool));
     std::shared_ptr<arrow::ArrayData> new_data = array->data()->Copy();
     new_data->type = read_type;
     new_data->child_data = {values->data()};
@@ -304,12 +306,12 @@ Result<std::shared_ptr<arrow::Array>> AlignListArrayByPaimonIds(
 }
 
 Result<std::shared_ptr<arrow::Array>> AlignMapArrayByPaimonIds(
-    const std::shared_ptr<arrow::MapArray>& array,
-    const std::shared_ptr<arrow::MapType>& read_type) {
+    const std::shared_ptr<arrow::MapArray>& array, const std::shared_ptr<arrow::MapType>& read_type,
+    arrow::MemoryPool* arrow_pool) {
     std::shared_ptr<arrow::Array> keys = array->keys();
-    PAIMON_ASSIGN_OR_RAISE(keys, AlignArrayByPaimonIds(keys, read_type->key_type()));
+    PAIMON_ASSIGN_OR_RAISE(keys, AlignArrayByPaimonIds(keys, read_type->key_type(), arrow_pool));
     std::shared_ptr<arrow::Array> items = array->items();
-    PAIMON_ASSIGN_OR_RAISE(items, AlignArrayByPaimonIds(items, read_type->item_type()));
+    PAIMON_ASSIGN_OR_RAISE(items, AlignArrayByPaimonIds(items, read_type->item_type(), arrow_pool));
 
     const std::shared_ptr<arrow::ArrayData>& entries_data = array->data()->child_data[0];
     std::shared_ptr<arrow::ArrayData> new_entries = entries_data->Copy();
@@ -323,7 +325,8 @@ Result<std::shared_ptr<arrow::Array>> AlignMapArrayByPaimonIds(
 }
 
 Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
-    const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& read_type) {
+    const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& read_type,
+    arrow::MemoryPool* arrow_pool) {
     if (array->type()->id() != read_type->id()) {
         return Status::Invalid(fmt::format("prepared value type {} does not match query type {}",
                                            array->type()->ToString(), read_type->ToString()));
@@ -331,13 +334,16 @@ Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
     switch (read_type->id()) {
         case arrow::Type::STRUCT:
             return AlignStructArrayByPaimonIds(checked_pointer_cast<arrow::StructArray>(array),
-                                               checked_pointer_cast<arrow::StructType>(read_type));
+                                               checked_pointer_cast<arrow::StructType>(read_type),
+                                               arrow_pool);
         case arrow::Type::LIST:
             return AlignListArrayByPaimonIds(checked_pointer_cast<arrow::ListArray>(array),
-                                             checked_pointer_cast<arrow::ListType>(read_type));
+                                             checked_pointer_cast<arrow::ListType>(read_type),
+                                             arrow_pool);
         case arrow::Type::MAP:
             return AlignMapArrayByPaimonIds(checked_pointer_cast<arrow::MapArray>(array),
-                                            checked_pointer_cast<arrow::MapType>(read_type));
+                                            checked_pointer_cast<arrow::MapType>(read_type),
+                                            arrow_pool);
         default:
             if (!array->type()->Equals(*read_type)) {
                 return Status::Invalid(
@@ -351,7 +357,7 @@ Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
 Result<arrow::ArrayVector> ProjectFieldsByPaimonIds(
     const std::shared_ptr<arrow::StructArray>& data_batch,
     const std::shared_ptr<arrow::Schema>& prepared_schema,
-    const std::shared_ptr<arrow::Schema>& query_schema) {
+    const std::shared_ptr<arrow::Schema>& query_schema, arrow::MemoryPool* arrow_pool) {
     std::unordered_map<int32_t, int32_t> prepared_field_id_to_idx;
     prepared_field_id_to_idx.reserve(prepared_schema->num_fields());
     for (int32_t i = kPreparedValueStartIndex; i < prepared_schema->num_fields(); ++i) {
@@ -375,7 +381,7 @@ Result<arrow::ArrayVector> ProjectFieldsByPaimonIds(
         }
         std::shared_ptr<arrow::Array> field_array = data_batch->field(prepared_iter->second);
         PAIMON_ASSIGN_OR_RAISE(field_array,
-                               AlignArrayByPaimonIds(field_array, query_field->type()));
+                               AlignArrayByPaimonIds(field_array, query_field->type(), arrow_pool));
         result.push_back(std::move(field_array));
     }
     return result;
@@ -468,8 +474,12 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
     };
 
     Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatch() override {
+        if (first_error_.has_value()) {
+            return first_error_.value();
+        }
         Result<std::unique_ptr<KeyValueRecordReader::Iterator>> result = NextBatchImpl();
         if (!result.ok()) {
+            first_error_ = result.status();
             Close();
         }
         return result;
@@ -508,6 +518,23 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
             }
             std::shared_ptr<arrow::StructArray> data_batch =
                 checked_pointer_cast<arrow::StructArray>(arrow_array);
+            Status transport_status =
+                ValidatePreparedTransportSchema(arrow::schema(data_batch->type()->fields()));
+            if (!transport_status.ok()) {
+                return Status::Invalid(
+                    "prepared batch field does not match prepared transport "
+                    "schema: ",
+                    transport_status.ToString());
+            }
+            if (visible_offsets_.has_value()) {
+                PAIMON_RETURN_NOT_OK(ValidateProjectionSchema(
+                    arrow::schema(data_batch->type()->fields()), key_schema_));
+                PAIMON_ASSIGN_OR_RAISE(
+                    arrow_array,
+                    AlignArrayByPaimonIds(data_batch, arrow::struct_(prepared_schema_->fields()),
+                                          arrow_pool_.get()));
+                data_batch = checked_pointer_cast<arrow::StructArray>(arrow_array);
+            }
             PAIMON_RETURN_NOT_OK(ValidatePreparedBatch(data_batch));
             PAIMON_RETURN_NOT_OK(ValidateOrdering(data_batch));
 
@@ -528,12 +555,12 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
                 data_batch->field(kValueKindIndex));
             sequence_number_array_ = checked_pointer_cast<arrow::NumericArray<arrow::Int64Type>>(
                 data_batch->field(kSequenceNumberIndex));
-            PAIMON_ASSIGN_OR_RAISE(
-                arrow::ArrayVector key_fields,
-                ProjectFieldsByPaimonIds(data_batch, prepared_schema_, key_schema_));
-            PAIMON_ASSIGN_OR_RAISE(
-                arrow::ArrayVector value_fields,
-                ProjectFieldsByPaimonIds(data_batch, prepared_schema_, value_schema_));
+            PAIMON_ASSIGN_OR_RAISE(arrow::ArrayVector key_fields,
+                                   ProjectFieldsByPaimonIds(data_batch, prepared_schema_,
+                                                            key_schema_, arrow_pool_.get()));
+            PAIMON_ASSIGN_OR_RAISE(arrow::ArrayVector value_fields,
+                                   ProjectFieldsByPaimonIds(data_batch, prepared_schema_,
+                                                            value_schema_, arrow_pool_.get()));
             key_ctx_ = std::make_shared<ColumnarBatchContext>(key_fields, pool_);
             value_ctx_ = std::make_shared<ColumnarBatchContext>(value_fields, pool_);
             ArrowUtils::TraverseArray(data_batch);
@@ -578,8 +605,9 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
         if (data_batch->length() == 0) {
             return Status::OK();
         }
-        PAIMON_ASSIGN_OR_RAISE(arrow::ArrayVector key_fields,
-                               ProjectFieldsByPaimonIds(data_batch, prepared_schema_, key_schema_));
+        PAIMON_ASSIGN_OR_RAISE(
+            arrow::ArrayVector key_fields,
+            ProjectFieldsByPaimonIds(data_batch, prepared_schema_, key_schema_, arrow_pool_.get()));
         std::shared_ptr<ColumnarBatchContext> key_context =
             std::make_shared<ColumnarBatchContext>(key_fields, pool_);
         std::shared_ptr<arrow::Int64Array> sequences =
@@ -613,6 +641,7 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
 
  private:
     bool closed_ = false;
+    std::optional<Status> first_error_;
     std::unique_ptr<BatchReader> reader_;
     std::shared_ptr<arrow::Schema> prepared_schema_;
     std::optional<OffsetRange> visible_offsets_;
@@ -634,6 +663,19 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
 
 }  // namespace
 
+Status ValidatePreparedTransportSchema(const std::shared_ptr<arrow::Schema>& prepared_schema) {
+    if (!prepared_schema || prepared_schema->num_fields() < kPreparedValueStartIndex) {
+        return Status::Invalid("prepared schema must contain realtime transport fields");
+    }
+    PAIMON_RETURN_NOT_OK(
+        CheckPreparedField(prepared_schema, kValueKindIndex, SpecialFields::ValueKind()));
+    PAIMON_RETURN_NOT_OK(
+        CheckPreparedField(prepared_schema, kSequenceNumberIndex, SpecialFields::SequenceNumber()));
+    PAIMON_RETURN_NOT_OK(
+        CheckPreparedField(prepared_schema, kRealtimeOffsetIndex, RealtimeOffsetField()));
+    return Status::OK();
+}
+
 namespace {
 
 Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderImpl(
@@ -649,7 +691,7 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderImpl(
         return Status::Invalid("prepared batch reader cannot be null");
     }
     ScopeGuard close_guard([&owned_reader]() -> void { owned_reader->Close(); });
-    PAIMON_RETURN_NOT_OK(ValidatePreparedSchema(prepared_schema));
+    PAIMON_RETURN_NOT_OK(ValidatePreparedTransportSchema(prepared_schema));
     if (!key_schema) {
         return Status::Invalid("prepared key schema cannot be null");
     }
@@ -695,26 +737,23 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> AdaptPreparedCommitBa
     const std::shared_ptr<arrow::Schema>& value_schema,
     const std::shared_ptr<FieldsComparator>& key_comparator,
     const std::shared_ptr<MemoryPool>& memory_pool) {
+    std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers;
+    ScopeGuard readers_guard([&readers, &adapted_readers]() {
+        CloseReaders(readers);
+        CloseReaders(adapted_readers);
+    });
     if (!memory_pool) {
         return Status::Invalid("prepared reader memory pool cannot be null");
     }
+    for (const std::unique_ptr<BatchReader>& reader : readers) {
+        if (!reader) {
+            return Status::Invalid("PK real-time store returned a null commit reader");
+        }
+    }
     std::shared_ptr<arrow::MemoryPool> arrow_pool = GetArrowPool(memory_pool);
     PAIMON_ASSIGN_OR_RAISE(
         std::shared_ptr<RealtimeOffsetCoverage> offset_coverage,
         RealtimeOffsetCoverage::Create(sealed_offsets, readers.size(), arrow_pool));
-    ScopeGuard readers_guard([&readers]() {
-        for (const std::unique_ptr<BatchReader>& reader : readers) {
-            if (reader) {
-                reader->Close();
-            }
-        }
-    });
-    std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers;
-    ScopeGuard adapted_readers_guard([&adapted_readers]() {
-        for (const std::unique_ptr<KeyValueRecordReader>& reader : adapted_readers) {
-            reader->Close();
-        }
-    });
     adapted_readers.reserve(readers.size());
     for (std::unique_ptr<BatchReader>& reader : readers) {
         PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> adapted_reader,
@@ -724,7 +763,6 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> AdaptPreparedCommitBa
         adapted_readers.push_back(std::move(adapted_reader));
     }
     readers_guard.Release();
-    adapted_readers_guard.Release();
     return adapted_readers;
 }
 
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.h b/src/paimon/core/realtime/prepared_key_value_reader.h
index 064a62958..22a837a76 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.h
+++ b/src/paimon/core/realtime/prepared_key_value_reader.h
@@ -33,6 +33,8 @@ class BatchReader;
 class FieldsComparator;
 class MemoryPool;
 
+Status ValidatePreparedTransportSchema(const std::shared_ptr<arrow::Schema>& prepared_schema);
+
 Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
     std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
     const std::optional<OffsetRange>& visible_offsets,
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index 2f04aae79..e4c480377 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -21,6 +21,8 @@
 #include <cstddef>
 #include <mutex>
 #include <optional>
+#include <queue>
+#include <unordered_map>
 #include <utility>
 #include <vector>
 
@@ -37,6 +39,7 @@
 #include "paimon/common/utils/fields_comparator.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/index/pk/primary_key_index_definitions.h"
+#include "paimon/core/realtime/prepared_key_value_reader.h"
 #include "paimon/core/schema/table_schema.h"
 #include "paimon/macros.h"
 
@@ -163,9 +166,12 @@ class RawBatchReader final : public BatchReader {
           key_comparator_(key_comparator),
           memory_pool_(memory_pool),
           arrow_pool_(GetArrowPool(memory_pool)),
+          heap_(SourceGreater{this}),
           metrics_(std::make_shared<MetricsImpl>()) {
         key_contexts_.reserve(batches_.size());
-        for (const StoredBatch& batch : batches_) {
+        sequence_arrays_.reserve(batches_.size());
+        for (size_t i = 0; i < batches_.size(); ++i) {
+            const StoredBatch& batch = batches_[i];
             arrow::ArrayVector key_arrays;
             key_arrays.reserve(key_field_indexes_.size());
             for (int32_t field_index : key_field_indexes_) {
@@ -173,34 +179,89 @@ class RawBatchReader final : public BatchReader {
             }
             key_contexts_.push_back(
                 std::make_shared<ColumnarBatchContext>(key_arrays, memory_pool_));
+            sequence_arrays_.push_back(
+                checked_pointer_cast<arrow::Int64Array>(batch.data->field(1)));
+            if (batch.data->length() > 0) {
+                heap_.push(i);
+            }
         }
     }
 
     Result<ReadBatch> NextBatch() override {
-        std::optional<size_t> selected;
-        for (size_t i = 0; i < batches_.size(); ++i) {
-            if (positions_[i] >= batches_[i].data->length()) {
-                continue;
+        if (heap_.empty()) {
+            return MakeEofBatch();
+        }
+
+        struct SelectedRow {
+            size_t selected_source;
+            int64_t source_ordinal;
+        };
+        struct SelectedSource {
+            size_t source;
+            std::vector<int64_t> rows;
+            int64_t base = -1;
+        };
+        std::vector<SelectedRow> selected_rows;
+        selected_rows.reserve(kOutputBatchSize);
+        std::vector<SelectedSource> selected_sources;
+        std::unordered_map<size_t, size_t> selected_source_indexes;
+        while (!heap_.empty() && selected_rows.size() < kOutputBatchSize) {
+            const size_t source = heap_.top();
+            heap_.pop();
+            auto [source_it, inserted] =
+                selected_source_indexes.emplace(source, selected_sources.size());
+            if (inserted) {
+                selected_sources.push_back(SelectedSource{source, {}});
             }
-            if (!selected.has_value() || Less(i, selected.value())) {
-                selected = i;
+            SelectedSource& selected_source = selected_sources[source_it->second];
+            selected_rows.push_back(
+                SelectedRow{source_it->second, static_cast<int64_t>(selected_source.rows.size())});
+            selected_source.rows.push_back(positions_[source]++);
+            if (positions_[source] < batches_[source].data->length()) {
+                heap_.push(source);
             }
         }
-        if (!selected.has_value()) {
-            return MakeEofBatch();
-        }
-        const size_t batch_index = selected.value();
-        arrow::Int64Builder index_builder(arrow_pool_.get());
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(index_builder.Append(positions_[batch_index]));
-        std::shared_ptr<arrow::Array> index;
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(index_builder.Finish(&index));
+
         arrow::compute::ExecContext context(arrow_pool_.get());
-        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-            arrow::Datum taken,
-            arrow::compute::Take(arrow::Datum(batches_[batch_index].data), arrow::Datum(index),
-                                 arrow::compute::TakeOptions::NoBoundsCheck(), &context));
-        std::shared_ptr<arrow::Array> batch = taken.make_array();
-        ++positions_[batch_index];
+        arrow::ArrayVector grouped_batches;
+        int64_t grouped_row_count = 0;
+        for (SelectedSource& selected_source : selected_sources) {
+            arrow::Int64Builder source_index_builder(arrow_pool_.get());
+            PAIMON_RETURN_NOT_OK_FROM_ARROW(
+                source_index_builder.AppendValues(selected_source.rows));
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> source_indices,
+                                              source_index_builder.Finish());
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+                arrow::Datum source_batch,
+                arrow::compute::Take(arrow::Datum(batches_[selected_source.source].data),
+                                     arrow::Datum(source_indices),
+                                     arrow::compute::TakeOptions::NoBoundsCheck(), &context));
+            selected_source.base = grouped_row_count;
+            grouped_row_count += static_cast<int64_t>(selected_source.rows.size());
+            grouped_batches.push_back(source_batch.make_array());
+        }
+
+        std::shared_ptr<arrow::Array> batch;
+        if (grouped_batches.size() == 1) {
+            batch = std::move(grouped_batches[0]);
+        } else {
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+                std::shared_ptr<arrow::Array> grouped,
+                arrow::Concatenate(grouped_batches, arrow_pool_.get()));
+            arrow::Int64Builder order_builder(arrow_pool_.get());
+            PAIMON_RETURN_NOT_OK_FROM_ARROW(order_builder.Reserve(selected_rows.size()));
+            for (const SelectedRow& selected : selected_rows) {
+                order_builder.UnsafeAppend(selected_sources[selected.selected_source].base +
+                                           selected.source_ordinal);
+            }
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> order,
+                                              order_builder.Finish());
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+                arrow::Datum reordered,
+                arrow::compute::Take(arrow::Datum(grouped), arrow::Datum(order),
+                                     arrow::compute::TakeOptions::NoBoundsCheck(), &context));
+            batch = reordered.make_array();
+        }
         auto array = std::make_unique<ArrowArray>();
         auto schema = std::make_unique<ArrowSchema>();
         PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*batch, array.get(), schema.get()));
@@ -211,12 +272,18 @@ class RawBatchReader final : public BatchReader {
         return metrics_;
     }
     void Close() override {
+        while (!heap_.empty()) {
+            heap_.pop();
+        }
         batches_.clear();
         positions_.clear();
         key_contexts_.clear();
+        sequence_arrays_.clear();
     }
 
  private:
+    static constexpr size_t kOutputBatchSize = 1024;
+
     bool Less(size_t left, size_t right) const {
         ColumnarRowRef left_key(key_contexts_[left], positions_[left]);
         ColumnarRowRef right_key(key_contexts_[right], positions_[right]);
@@ -224,13 +291,22 @@ class RawBatchReader final : public BatchReader {
         if (key_comparison != 0) {
             return key_comparison < 0;
         }
-        const std::shared_ptr<arrow::Int64Array> left_sequences =
-            checked_pointer_cast<arrow::Int64Array>(batches_[left].data->field(1));
-        const std::shared_ptr<arrow::Int64Array> right_sequences =
-            checked_pointer_cast<arrow::Int64Array>(batches_[right].data->field(1));
-        return left_sequences->Value(positions_[left]) < right_sequences->Value(positions_[right]);
+        const int64_t left_sequence = sequence_arrays_[left]->Value(positions_[left]);
+        const int64_t right_sequence = sequence_arrays_[right]->Value(positions_[right]);
+        if (left_sequence != right_sequence) {
+            return left_sequence < right_sequence;
+        }
+        return left < right;
     }
 
+    struct SourceGreater {
+        RawBatchReader* reader;
+
+        bool operator()(size_t left, size_t right) const {
+            return reader->Less(right, left);
+        }
+    };
+
     std::vector<StoredBatch> batches_;
     std::vector<int64_t> positions_;
     std::vector<int32_t> key_field_indexes_;
@@ -238,6 +314,8 @@ class RawBatchReader final : public BatchReader {
     std::shared_ptr<MemoryPool> memory_pool_;
     std::shared_ptr<arrow::MemoryPool> arrow_pool_;
     std::vector<std::shared_ptr<ColumnarBatchContext>> key_contexts_;
+    std::vector<std::shared_ptr<arrow::Int64Array>> sequence_arrays_;
+    std::priority_queue<size_t, std::vector<size_t>, SourceGreater> heap_;
     std::shared_ptr<Metrics> metrics_;
 };
 
@@ -379,8 +457,9 @@ Result<std::shared_ptr<PrimaryKeyRealtimeStore>> PrimaryKeyRealtimeStore::Create
     const std::shared_ptr<arrow::Schema>& prepared_schema,
     const std::vector<std::string>& trimmed_primary_keys,
     const std::shared_ptr<MemoryPool>& memory_pool) {
-    if (!prepared_schema || trimmed_primary_keys.empty() || !memory_pool) {
-        return Status::Invalid("PK prepared schema or memory pool is null");
+    PAIMON_RETURN_NOT_OK(ValidatePreparedTransportSchema(prepared_schema));
+    if (trimmed_primary_keys.empty() || !memory_pool) {
+        return Status::Invalid("PK primary keys are empty or memory pool is null");
     }
     std::vector<int32_t> key_field_indexes;
     std::vector<DataField> key_fields;
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index 116c6e389..dc2ce86b1 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -18,14 +18,18 @@
 
 #include "paimon/core/realtime/primary_key_realtime_store.h"
 
+#include <algorithm>
 #include <memory>
+#include <numeric>
 #include <optional>
 #include <string>
+#include <utility>
 #include <vector>
 
 #include "arrow/api.h"
 #include "arrow/c/bridge.h"
 #include "arrow/ipc/json_simple.h"
+#include "fmt/format.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/arrow/status_utils.h"
@@ -196,6 +200,34 @@ TEST(PrimaryKeyRealtimeStoreTest, TestWriteAndSealValidation) {
         RealtimeWriteBatch{MakeBatch(R"([[0, 4, 3, 4, "four"]])"), OffsetRange(3, 4)}));
 }
 
+TEST(PrimaryKeyRealtimeStoreTest, TestBadTransportPrefix) {
+    const std::shared_ptr<arrow::Schema> valid = PreparedSchema();
+    std::vector<arrow::FieldVector> invalid_fields;
+
+    arrow::FieldVector wrong_type = valid->fields();
+    wrong_type[0] = DataField::ConvertDataFieldToArrowField(
+                        DataField(SpecialFields::ValueKind().Id(),
+                                  arrow::field("_VALUE_KIND", arrow::int32(), false)))
+                        ->WithNullable(false);
+    invalid_fields.push_back(std::move(wrong_type));
+
+    arrow::FieldVector nullable_sequence = valid->fields();
+    nullable_sequence[1] = nullable_sequence[1]->WithNullable(true);
+    invalid_fields.push_back(std::move(nullable_sequence));
+
+    arrow::FieldVector wrong_offset_id = valid->fields();
+    wrong_offset_id[2] = DataField::ConvertDataFieldToArrowField(
+                             DataField(99, arrow::field("_REALTIME_OFFSET", arrow::int64(), false)))
+                             ->WithNullable(false);
+    invalid_fields.push_back(std::move(wrong_offset_id));
+
+    for (const arrow::FieldVector& fields : invalid_fields) {
+        ASSERT_NOK_WITH_MSG(
+            PrimaryKeyRealtimeStore::Create(arrow::schema(fields), {"id"}, GetDefaultPool()),
+            "prepared schema field");
+    }
+}
+
 TEST(PrimaryKeyRealtimeStoreTest, TestCommitBatches) {
     ASSERT_OK_AND_ASSIGN(
         std::shared_ptr<PrimaryKeyRealtimeStore> store,
@@ -233,12 +265,100 @@ TEST(PrimaryKeyRealtimeStoreTest, TestCommitReaderExportsZeroOffsets) {
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
                          store->CreateCommitReaders(segment.value()));
     ASSERT_EQ(1, readers.size());
-    for (int32_t row = 0; row < 2; ++row) {
+    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, readers[0]->NextBatch());
+    ASSERT_FALSE(BatchReader::IsEofBatch(batch));
+    ASSERT_EQ(2, batch.first->length);
+    AssertOffsetsZero(batch.first.get());
+    ASSERT_TRUE(arrow::ImportArray(batch.first.get(), batch.second.get()).ok());
+    ASSERT_OK_AND_ASSIGN(batch, readers[0]->NextBatch());
+    ASSERT_TRUE(BatchReader::IsEofBatch(batch));
+}
+
+TEST(PrimaryKeyRealtimeStoreTest, TestHeapMergeAcrossBatches) {
+    constexpr int64_t kSourceCount = 2057;
+    constexpr int64_t kKeyCount = 257;
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr<PrimaryKeyRealtimeStore> store,
+        PrimaryKeyRealtimeStore::Create(PreparedSchema(), {"id"}, GetDefaultPool()));
+    for (int64_t source = 0; source < kSourceCount; ++source) {
+        const int64_t id = (source * 149) % kKeyCount;
+        const std::string json =
+            fmt::format(R"([[0, {}, {}, {}, "v{}"]])", source, source, id, source);
+        ASSERT_OK(
+            store->Write(RealtimeWriteBatch{MakeBatch(json), OffsetRange(source, source + 1)}));
+    }
+    ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
+                         store->SealForCommit());
+    ASSERT_TRUE(segment.has_value());
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store->CreateCommitReaders(segment.value()));
+    ASSERT_EQ(1, readers.size());
+
+    std::vector<int64_t> expected_sources(kSourceCount);
+    std::iota(expected_sources.begin(), expected_sources.end(), 0);
+    std::sort(expected_sources.begin(), expected_sources.end(), [=](int64_t left, int64_t right) {
+        const int64_t left_id = (left * 149) % kKeyCount;
+        const int64_t right_id = (right * 149) % kKeyCount;
+        return left_id != right_id ? left_id < right_id : left < right;
+    });
+
+    int64_t output_row = 0;
+    int64_t output_batches = 0;
+    while (true) {
         ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, readers[0]->NextBatch());
-        ASSERT_FALSE(BatchReader::IsEofBatch(batch));
-        AssertOffsetsZero(batch.first.get());
-        ASSERT_TRUE(arrow::ImportArray(batch.first.get(), batch.second.get()).ok());
+        if (BatchReader::IsEofBatch(batch)) {
+            break;
+        }
+        ASSERT_LE(batch.first->length, 1024);
+        ASSERT_GT(batch.first->length, 0);
+        ++output_batches;
+        arrow::Result<std::shared_ptr<arrow::Array>> imported_result =
+            arrow::ImportArray(batch.first.get(), batch.second.get());
+        ASSERT_TRUE(imported_result.ok()) << imported_result.status().ToString();
+        std::shared_ptr<arrow::Array> imported = std::move(imported_result).ValueOrDie();
+        std::shared_ptr<arrow::StructArray> array =
+            std::dynamic_pointer_cast<arrow::StructArray>(imported);
+        ASSERT_NE(nullptr, array);
+        ASSERT_EQ(PreparedSchema()->ToString(), arrow::schema(array->type()->fields())->ToString());
+        std::shared_ptr<arrow::Int64Array> sequences =
+            std::dynamic_pointer_cast<arrow::Int64Array>(array->field(1));
+        std::shared_ptr<arrow::Int64Array> ids =
+            std::dynamic_pointer_cast<arrow::Int64Array>(array->field(3));
+        std::shared_ptr<arrow::StringArray> values =
+            std::dynamic_pointer_cast<arrow::StringArray>(array->field(4));
+        ASSERT_NE(nullptr, sequences);
+        ASSERT_NE(nullptr, ids);
+        ASSERT_NE(nullptr, values);
+        for (int64_t row = 0; row < array->length(); ++row, ++output_row) {
+            ASSERT_LT(output_row, kSourceCount);
+            const int64_t source = expected_sources[output_row];
+            ASSERT_EQ(source, sequences->Value(row));
+            ASSERT_EQ((source * 149) % kKeyCount, ids->Value(row));
+            ASSERT_EQ(fmt::format("v{}", source), values->GetString(row));
+        }
     }
+    ASSERT_EQ(kSourceCount, output_row);
+    ASSERT_EQ(3, output_batches);
+}
+
+TEST(PrimaryKeyRealtimeStoreTest, TestCloseUnreadMultiSourceReader) {
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr<PrimaryKeyRealtimeStore> store,
+        PrimaryKeyRealtimeStore::Create(PreparedSchema(), {"id"}, GetDefaultPool()));
+    ASSERT_OK(
+        store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 10, 0, 1, "a"]])"), OffsetRange(0, 1)}));
+    ASSERT_OK(
+        store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 20, 1, 2, "b"]])"), OffsetRange(1, 2)}));
+    ASSERT_OK(
+        store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 30, 2, 3, "c"]])"), OffsetRange(2, 3)}));
+    ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
+                         store->SealForCommit());
+    ASSERT_TRUE(segment.has_value());
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store->CreateCommitReaders(segment.value()));
+    ASSERT_EQ(1, readers.size());
+
+    readers[0]->Close();
 }
 
 TEST(PrimaryKeyRealtimeStoreTest, TestReclaimKeepsReadView) {
diff --git a/src/paimon/core/realtime/realtime_fields.h b/src/paimon/core/realtime/realtime_fields.h
index 6ed04b38a..270941238 100644
--- a/src/paimon/core/realtime/realtime_fields.h
+++ b/src/paimon/core/realtime/realtime_fields.h
@@ -19,17 +19,15 @@
 
 #pragma once
 
-#include <cstdint>
-#include <limits>
-
 #include "arrow/type.h"
 #include "paimon/common/types/data_field.h"
+#include "paimon/utils/special_field_ids.h"
 
 namespace paimon {
 
 inline const DataField& RealtimeOffsetField() {
     static const DataField data_field =
-        DataField(std::numeric_limits<int32_t>::max() - 10002,
+        DataField(SpecialFieldIds::REALTIME_OFFSET,
                   arrow::field("_REALTIME_OFFSET", arrow::int64(), /*nullable=*/false));
     return data_field;
 }
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index 82318eadb..185156eb7 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -32,7 +32,6 @@
 #include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
-#include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/io/merged_key_value_record_reader.h"
 #include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
 #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
@@ -285,18 +284,6 @@ Status RealtimePrimaryKeyWriter::FlushSegment(const std::shared_ptr<RealtimeSegm
                                               const OffsetRange& sealed_offsets) {
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
                            realtime_store_->CreateCommitReaders(segment));
-    ScopeGuard readers_guard([&readers]() {
-        for (const std::unique_ptr<BatchReader>& reader : readers) {
-            if (reader) {
-                reader->Close();
-            }
-        }
-    });
-    for (const std::unique_ptr<BatchReader>& reader : readers) {
-        if (!reader) {
-            return Status::Invalid("PK real-time store returned a null commit reader");
-        }
-    }
     PAIMON_ASSIGN_OR_RAISE(
         std::vector<std::unique_ptr<KeyValueRecordReader>> prepared_readers,
         AdaptPreparedCommitBatchReaders(std::move(readers), prepared_schema_, sealed_offsets,
@@ -309,7 +296,6 @@ Status RealtimePrimaryKeyWriter::FlushSegment(const std::shared_ptr<RealtimeSegm
             std::move(prepared_reader), key_comparator_,
             std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge_function))));
     }
-    readers_guard.Release();
     return merge_tree_writer_->WriteSortedReaders(std::move(sorted_readers));
 }
 
diff --git a/src/paimon/core/table/source/append_only_table_read.cpp b/src/paimon/core/table/source/append_only_table_read.cpp
index 6885dc374..34c6ef850 100644
--- a/src/paimon/core/table/source/append_only_table_read.cpp
+++ b/src/paimon/core/table/source/append_only_table_read.cpp
@@ -77,6 +77,13 @@ Result<std::unique_ptr<BatchReader>> AppendOnlyTableRead::CreateReader(
     std::vector<std::unique_ptr<BatchReader>> readers;
     readers.reserve(splits.size());
     std::vector<std::shared_ptr<RealtimeSplit>> realtime_splits;
+    ScopeGuard cleanup_guard([&]() {
+        for (const std::unique_ptr<BatchReader>& reader : readers) {
+            if (reader) {
+                reader->Close();
+            }
+        }
+    });
     for (const std::shared_ptr<Split>& split : splits) {
         std::shared_ptr<RealtimeSplit> realtime_split =
             std::dynamic_pointer_cast<RealtimeSplit>(split);
@@ -92,8 +99,6 @@ Result<std::unique_ptr<BatchReader>> AppendOnlyTableRead::CreateReader(
         }
     }
 
-    std::unique_ptr<BatchReader> result =
-        std::make_unique<ConcatBatchReader>(std::move(readers), GetMemoryPool());
     if (!realtime_splits.empty()) {
         const std::shared_ptr<RealtimeContext> realtime_context = context_->GetRealtimeContext();
         if (!realtime_context) {
@@ -106,6 +111,9 @@ Result<std::unique_ptr<BatchReader>> AppendOnlyTableRead::CreateReader(
                 realtime_context_impl->ReleaseReadView(realtime_split->OpaqueTicket()));
         }
     }
+    std::unique_ptr<BatchReader> result =
+        std::make_unique<ConcatBatchReader>(std::move(readers), GetMemoryPool());
+    cleanup_guard.Release();
     return result;
 }
 
@@ -124,6 +132,13 @@ Result<std::unique_ptr<BatchReader>> AppendOnlyTableRead::CreateRealtimeReader(
                            realtime_context_impl->ResolveReadView(realtime_split->OpaqueTicket()));
     std::vector<std::unique_ptr<BatchReader>> readers;
     readers.reserve(realtime_split->DiskSplits().size() + 1);
+    ScopeGuard readers_guard([&readers]() {
+        for (const std::unique_ptr<BatchReader>& reader : readers) {
+            if (reader) {
+                reader->Close();
+            }
+        }
+    });
     const RealtimePartitionBucket expected_partition_bucket(realtime_split->Partition(),
                                                             realtime_split->Bucket());
     if (memory.partition_bucket != expected_partition_bucket) {
@@ -150,8 +165,17 @@ Result<std::unique_ptr<BatchReader>> AppendOnlyTableRead::CreateRealtimeReader(
         std::vector<std::unique_ptr<BatchReader>> memory_readers,
         memory.store->CreateQueryReaders(memory.read_view, realtime_split->CommittedEndOffset(),
                                          query_context));
-
+    const size_t first_memory_reader = readers.size();
+    readers.reserve(readers.size() + memory_readers.size());
     for (std::unique_ptr<BatchReader>& memory_reader : memory_readers) {
+        readers.push_back(std::move(memory_reader));
+    }
+
+    for (size_t i = first_memory_reader; i < readers.size(); ++i) {
+        std::unique_ptr<BatchReader>& memory_reader = readers[i];
+        if (!memory_reader) {
+            return Status::Invalid("append-only real-time store returned a null query reader");
+        }
         if (context_->EnablePredicateFilter() && context_->GetPredicate()) {
             PAIMON_ASSIGN_OR_RAISE(memory_reader, PredicateBatchReader::Create(
                                                       std::move(memory_reader),
@@ -159,14 +183,15 @@ Result<std::unique_ptr<BatchReader>> AppendOnlyTableRead::CreateRealtimeReader(
         }
         PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RealtimeReader> realtime_reader,
                                RealtimeReader::Create(memory.read_view, std::move(memory_reader)));
-        readers.push_back(std::move(realtime_reader));
+        memory_reader = std::move(realtime_reader);
     }
-    std::unique_ptr<BatchReader> result =
-        std::make_unique<ConcatBatchReader>(std::move(readers), GetMemoryPool());
     if (release_ticket) {
         PAIMON_RETURN_NOT_OK(
             realtime_context_impl->ReleaseReadView(realtime_split->OpaqueTicket()));
     }
+    std::unique_ptr<BatchReader> result =
+        std::make_unique<ConcatBatchReader>(std::move(readers), GetMemoryPool());
+    readers_guard.Release();
     return result;
 }
 
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index 64d722097..96f3f00d9 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -209,6 +209,13 @@ Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateReader(
     std::vector<std::unique_ptr<BatchReader>> readers;
     readers.reserve(splits.size());
     std::vector<std::shared_ptr<RealtimeSplit>> realtime_splits;
+    ScopeGuard cleanup_guard([&]() {
+        for (const std::unique_ptr<BatchReader>& reader : readers) {
+            if (reader) {
+                reader->Close();
+            }
+        }
+    });
     for (const std::shared_ptr<Split>& split : splits) {
         std::shared_ptr<RealtimeSplit> realtime_split =
             std::dynamic_pointer_cast<RealtimeSplit>(split);
@@ -223,8 +230,6 @@ Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateReader(
         }
     }
 
-    std::unique_ptr<BatchReader> result =
-        std::make_unique<ConcatBatchReader>(std::move(readers), GetMemoryPool());
     if (!realtime_splits.empty()) {
         const std::shared_ptr<RealtimeContext> realtime_context = context_->GetRealtimeContext();
         if (!realtime_context) {
@@ -237,6 +242,9 @@ Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateReader(
                 realtime_context_impl->ReleaseReadView(realtime_split->OpaqueTicket()));
         }
     }
+    std::unique_ptr<BatchReader> result =
+        std::make_unique<ConcatBatchReader>(std::move(readers), GetMemoryPool());
+    cleanup_guard.Release();
     return result;
 }
 
diff --git a/src/paimon/core/table/source/realtime_table_scan.cpp b/src/paimon/core/table/source/realtime_table_scan.cpp
index 1b496d8a1..4c3968dc3 100644
--- a/src/paimon/core/table/source/realtime_table_scan.cpp
+++ b/src/paimon/core/table/source/realtime_table_scan.cpp
@@ -152,16 +152,17 @@ Result<std::vector<std::shared_ptr<Split>>> RealtimeTableScan::CreateRealtimeSpl
             continue;
         }
 
-        // Append tables can schedule all but the tail disk split independently. The tail split
-        // carries the immutable memory view so disk and memory are still concatenated by one
-        // RealtimeSplit without collapsing the whole partition-bucket into one scheduling unit.
-        auto tail_disk_split = std::prev(grouped_disk_splits.end());
-        result.insert(result.end(), grouped_disk_splits.begin(), tail_disk_split);
-        std::vector<std::shared_ptr<Split>> realtime_disk_splits;
-        realtime_disk_splits.push_back(std::move(*tail_disk_split));
         RealtimePartitionBucketView& memory = memory_iter->second;
+        if (!pk_table_) {
+            // Append tables can schedule all but the tail disk split independently. The tail split
+            // carries the immutable memory view so disk and memory are still concatenated by one
+            // RealtimeSplit without collapsing the whole partition-bucket into one scheduling unit.
+            auto tail_disk_split = std::prev(grouped_disk_splits.end());
+            result.insert(result.end(), grouped_disk_splits.begin(), tail_disk_split);
+            grouped_disk_splits.erase(grouped_disk_splits.begin(), tail_disk_split);
+        }
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Split> realtime_split,
-                               create_realtime_split(key, std::move(realtime_disk_splits), memory));
+                               create_realtime_split(key, std::move(grouped_disk_splits), memory));
         result.push_back(std::move(realtime_split));
         active_memory.erase(memory_iter);
     }
@@ -176,7 +177,7 @@ Result<std::vector<std::shared_ptr<Split>>> RealtimeTableScan::CreateRealtimeSpl
     return result;
 }
 
-RealtimeTableScan::RealtimeTableScan(std::unique_ptr<TableScan>&& disk_scan,
+RealtimeTableScan::RealtimeTableScan(std::unique_ptr<TableScan>&& disk_scan, bool pk_table,
                                      const std::shared_ptr<RealtimeContextImpl>& realtime_context,
                                      const std::shared_ptr<FileStorePathFactory>& path_factory,
                                      const std::shared_ptr<SnapshotManager>& snapshot_manager,
@@ -184,6 +185,7 @@ RealtimeTableScan::RealtimeTableScan(std::unique_ptr<TableScan>&& disk_scan,
                                      const std::shared_ptr<ScanFilter>& scan_filter,
                                      int64_t read_view_ttl_millis)
     : disk_scan_(std::move(disk_scan)),
+      pk_table_(pk_table),
       realtime_context_(realtime_context),
       path_factory_(path_factory),
       snapshot_manager_(snapshot_manager),
diff --git a/src/paimon/core/table/source/realtime_table_scan.h b/src/paimon/core/table/source/realtime_table_scan.h
index 959203ca4..692b749ef 100644
--- a/src/paimon/core/table/source/realtime_table_scan.h
+++ b/src/paimon/core/table/source/realtime_table_scan.h
@@ -38,7 +38,7 @@ class SnapshotManager;
 /// Adds process-local memory splits to a normal data-table batch scan.
 class RealtimeTableScan : public TableScan {
  public:
-    RealtimeTableScan(std::unique_ptr<TableScan>&& disk_scan,
+    RealtimeTableScan(std::unique_ptr<TableScan>&& disk_scan, bool pk_table,
                       const std::shared_ptr<RealtimeContextImpl>& realtime_context,
                       const std::shared_ptr<FileStorePathFactory>& path_factory,
                       const std::shared_ptr<SnapshotManager>& snapshot_manager,
@@ -67,6 +67,7 @@ class RealtimeTableScan : public TableScan {
         const std::optional<int64_t>& snapshot_id) const;
 
     std::unique_ptr<TableScan> disk_scan_;
+    bool pk_table_;
     std::shared_ptr<RealtimeContextImpl> realtime_context_;
     std::shared_ptr<FileStorePathFactory> path_factory_;
     std::shared_ptr<SnapshotManager> snapshot_manager_;
diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp
index 0bcd79f61..f894e1a74 100644
--- a/src/paimon/core/table/source/table_scan.cpp
+++ b/src/paimon/core/table/source/table_scan.cpp
@@ -344,7 +344,7 @@ Result<std::unique_ptr<TableScan>> NewDataTableScan(const std::shared_ptr<ScanCo
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context,
                                RealtimeContextImpl::Cast(context->GetRealtimeContext()));
         return std::make_unique<RealtimeTableScan>(
-            std::move(batch_scan), realtime_context, path_factory,
+            std::move(batch_scan), pk_table, realtime_context, path_factory,
             snapshot_reader->GetSnapshotManager(), core_options.GetFileSystem(),
             context->GetScanFilters(), core_options.GetRealtimeReadViewTtlMillis());
     }
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index 0e6f83b70..a393f838d 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -468,21 +468,27 @@ class SwapFirstTwoBatchReader final : public BatchReader {
     Result<ReadBatch> NextBatch() override {
         if (!initialized_) {
             initialized_ = true;
-            PAIMON_ASSIGN_OR_RAISE(ReadBatch first, delegate_->NextBatch());
-            if (BatchReader::IsEofBatch(first)) {
+            PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, delegate_->NextBatch());
+            if (BatchReader::IsEofBatch(batch)) {
                 return MakeEofBatch();
             }
-            PAIMON_ASSIGN_OR_RAISE(ReadBatch second, delegate_->NextBatch());
-            if (BatchReader::IsEofBatch(second)) {
-                return first;
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+                std::shared_ptr<arrow::Array> array,
+                arrow::ImportArray(batch.first.get(), batch.second.get()));
+            if (array->length() < 2) {
+                return Status::Invalid("cannot make a one-row reader unsorted");
+            }
+            arrow::ArrayVector pieces = {array->Slice(1, 1), array->Slice(0, 1)};
+            if (array->length() > 2) {
+                pieces.push_back(array->Slice(2));
             }
-            first_ = std::move(first);
-            return second;
-        }
-        if (first_.has_value()) {
-            ReadBatch first = std::move(first_.value());
-            first_.reset();
-            return first;
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> swapped,
+                                              arrow::Concatenate(pieces));
+            auto output = std::make_unique<ArrowArray>();
+            auto schema = std::make_unique<ArrowSchema>();
+            PAIMON_RETURN_NOT_OK_FROM_ARROW(
+                arrow::ExportArray(*swapped, output.get(), schema.get()));
+            return ReadBatch(std::move(output), std::move(schema));
         }
         return delegate_->NextBatch();
     }
@@ -492,14 +498,12 @@ class SwapFirstTwoBatchReader final : public BatchReader {
     }
 
     void Close() override {
-        first_.reset();
         delegate_->Close();
     }
 
  private:
     bool initialized_ = false;
     std::unique_ptr<BatchReader> delegate_;
-    std::optional<ReadBatch> first_;
 };
 
 class SubstituteOffsetBatchReader final : public BatchReader {
@@ -1253,6 +1257,8 @@ class RealtimeWriteInteTest : public ::testing::Test {
         return Status::OK();
     }
 
+    void RunConcurrencyTest(bool primary_key);
+
     Result<RealtimeOffsetMap> ReadCommittedOffsets() const {
         PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(options_));
         SnapshotManager snapshot_manager(options.GetFileSystem(), table_path_);
@@ -1326,8 +1332,10 @@ class RealtimeWriteInteTest : public ::testing::Test {
         } else {
             CreateTable(/*partition_keys=*/{"pt"});
         }
+        auto close_state = std::make_shared<CloseTrackingReaderState>();
+        auto factory = std::make_shared<CloseTrackingRealtimeStoreFactory>(close_state);
         ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                             RealtimeContext::Create());
+                             RealtimeContext::Create(factory));
         ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                              CreateRealtimeWriter(realtime_context));
         std::vector<Row> p0_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
@@ -1363,6 +1371,9 @@ class RealtimeWriteInteTest : public ::testing::Test {
                              TableRead::Create(std::move(read_context)));
         ASSERT_NOK_WITH_MSG(table_read->CreateReader(invalid_splits),
                             "unsupported real-time split version");
+        if (!primary_key) {
+            ASSERT_EQ(1, close_state->query_close_count->load(std::memory_order_acquire));
+        }
 
         std::vector<Row> expected_rows = p0_rows;
         expected_rows.insert(expected_rows.end(), p1_rows.begin(), p1_rows.end());
@@ -1631,6 +1642,56 @@ TEST_F(RealtimeWriteInteTest, TestPkMergeDiskSealedAndActive) {
     ASSERT_OK(writer->Close());
 }
 
+TEST_F(RealtimeWriteInteTest, TestPkMergeAllDiskSplitsWithMemory) {
+    options_[Options::SOURCE_SPLIT_OPEN_FILE_COST] = "1";
+    options_[Options::SOURCE_SPLIT_TARGET_SIZE] = "1";
+    CreatePkTable();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+
+    const std::vector<std::vector<Row>> disk_batches = {
+        {{1, "disk-1", "p0"}, {2, "disk-2", "p0"}},
+        {{10, "disk-10", "p0"}, {11, "disk-11", "p0"}},
+        {{20, "disk-20", "p0"}, {21, "disk-21", "p0"}},
+    };
+    for (int64_t commit_identifier = 0;
+         commit_identifier < static_cast<int64_t>(disk_batches.size()); ++commit_identifier) {
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                             MakeBatch(disk_batches[commit_identifier], /*partitioned=*/false));
+        ASSERT_OK(writer->Write(std::move(batch)));
+        ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+                             writer->PrepareCommitWithProgress(commit_identifier));
+        ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, commit_identifier));
+        ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
+    }
+
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<RecordBatch> memory_batch,
+        MakeBatch({Row{1, "memory-1", "p0"}, Row{10, "deleted-10", "p0"}},
+                  /*partitioned=*/false, /*bucket=*/0,
+                  {RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::DELETE}));
+    ASSERT_OK(writer->Write(std::move(memory_batch)));
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
+                         CreatePlan(realtime_context, /*predicate=*/nullptr));
+    ASSERT_EQ(1, plan->Splits().size());
+    std::shared_ptr<RealtimeSplit> realtime_split =
+        std::dynamic_pointer_cast<RealtimeSplit>(plan->Splits()[0]);
+    ASSERT_NE(nullptr, realtime_split);
+    ASSERT_EQ(3, realtime_split->DiskSplits().size());
+
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(plan, realtime_context));
+    ASSERT_EQ((std::vector<Row>{{1, "memory-1", "p0"},
+                                {2, "disk-2", "p0"},
+                                {11, "disk-11", "p0"},
+                                {20, "disk-20", "p0"},
+                                {21, "disk-21", "p0"}}),
+              actual_rows);
+    ASSERT_OK(writer->Close());
+}
+
 TEST_F(RealtimeWriteInteTest, TestPkNestedProjectionAcrossDiskAndMemory) {
     const std::shared_ptr<arrow::Field> projected_b = arrow::field("b", arrow::int64());
     fields_ = {
@@ -1712,6 +1773,45 @@ TEST_F(RealtimeWriteInteTest, TestPkNestedProjectionAcrossDiskAndMemory) {
     ASSERT_OK(writer->Close());
 }
 
+TEST_F(RealtimeWriteInteTest, TestPkMemoryReadAfterSchemaEvolution) {
+    CreatePkTable();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         MakeBatch({Row{1, "old", "p0"}}, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(batch)));
+
+    std::shared_ptr<arrow::Field> renamed_payload = arrow::field("renamed_payload", arrow::utf8());
+    std::shared_ptr<arrow::Field> added = arrow::field("added", arrow::int32());
+    ASSERT_OK(TestHelper::WriteNextSchema(dir_->GetFileSystem(), table_path_,
+                                          {DataField(0, fields_[0]), DataField(1, renamed_payload),
+                                           DataField(2, fields_[2]), DataField(3, added)},
+                                          /*highest_field_id=*/3, options_));
+    fields_[1] = renamed_payload;
+    fields_.push_back(added);
+    schema_ = arrow::schema(fields_);
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
+                         CreatePlan(realtime_context, /*predicate=*/nullptr));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult result,
+                         ReadPlan(plan, realtime_context, {"id", "renamed_payload", "pt", "added"},
+                                  /*predicate=*/nullptr, /*enable_predicate_filter=*/false));
+    ASSERT_EQ(1, result.data->num_chunks());
+    std::shared_ptr<arrow::StructArray> row =
+        std::dynamic_pointer_cast<arrow::StructArray>(result.data->chunk(0));
+    ASSERT_NE(nullptr, row);
+    ASSERT_EQ(1, row->length());
+    std::shared_ptr<arrow::StringArray> renamed_values =
+        std::dynamic_pointer_cast<arrow::StringArray>(row->field(2));
+    ASSERT_NE(nullptr, renamed_values);
+    ASSERT_EQ("old", renamed_values->GetString(0));
+    ASSERT_TRUE(row->field(4)->IsNull(0));
+    result.reader->Close();
+    ASSERT_OK(writer->Close());
+}
+
 TEST_F(RealtimeWriteInteTest, TestPkCompositeMerge) {
     CreatePkTable(/*partition_keys=*/{}, /*primary_keys=*/{"id", "payload"});
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
@@ -2277,6 +2377,40 @@ TEST_F(RealtimeWriteInteTest, TestPkQueryReaderCloseFailure) {
     ASSERT_OK(writer->Close());
 }
 
+TEST_F(RealtimeWriteInteTest, TestAppendQueryReaderCloseFailure) {
+    CreateTable(/*partition_keys=*/{});
+    auto state = std::make_shared<CloseTrackingReaderState>();
+    state->query_null_index = 1;
+    auto factory = std::make_shared<CloseTrackingRealtimeStoreFactory>(state);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create(factory));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+    std::vector<Row> rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         MakeBatch(rows, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(batch)));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
+                         CreatePlan(realtime_context, /*predicate=*/nullptr));
+
+    ReadContextBuilder read_builder(table_path_);
+    read_builder.SetOptions(options_)
+        .SetReadFieldNames({"id", "payload", "pt"})
+        .WithRealtimeContext(realtime_context)
+        .WithMemoryPool(pool_);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableRead> table_read,
+                         TableRead::Create(std::move(read_context)));
+    ASSERT_NOK_WITH_MSG(table_read->CreateReader(plan->Splits()),
+                        "append-only real-time store returned a null query reader");
+    ASSERT_EQ(1, state->query_close_count->load(std::memory_order_acquire));
+
+    state->query_null_index = -1;
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(plan, realtime_context));
+    ASSERT_EQ(rows, actual_rows);
+    ASSERT_OK(writer->Close());
+}
+
 TEST_F(RealtimeWriteInteTest, TestPkCommitReaderCloseFailure) {
     CreatePkTable();
     auto state = std::make_shared<CloseTrackingReaderState>();
@@ -3705,8 +3839,12 @@ TEST_F(RealtimeWriteInteTest, TestReopenRealtimeContextAfterRollback) {
     ASSERT_OK(writer->Close());
 }
 
-TEST_F(RealtimeWriteInteTest, TestConcurrentWritePrepareCommitReadAndRefresh) {
-    CreateTable(/*partition_keys=*/{});
+void RealtimeWriteInteTest::RunConcurrencyTest(bool primary_key) {
+    if (primary_key) {
+        CreatePkTable();
+    } else {
+        CreateTable(/*partition_keys=*/{});
+    }
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                          RealtimeContext::Create());
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
@@ -3942,6 +4080,14 @@ TEST_F(RealtimeWriteInteTest, TestConcurrentWritePrepareCommitReadAndRefresh) {
     ASSERT_OK(writer->Close());
 }
 
+TEST_F(RealtimeWriteInteTest, TestConcurrentWritePrepareCommitReadAndRefresh) {
+    RunConcurrencyTest(/*primary_key=*/false);
+}
+
+TEST_F(RealtimeWriteInteTest, TestPkConcurrency) {
+    RunConcurrencyTest(/*primary_key=*/true);
+}
+
 TEST_F(RealtimeWriteInteTest, TestMultiplePartitions) {
     CreateTable(/*partition_keys=*/{"pt"});
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,

From d818466ba925ed719694bdb9ed389cefc730f825 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Tue, 25 Aug 2026 00:31:55 +0800
Subject: [PATCH 23/62] refactor(realtime): simplify primary-key contracts

---
 .../realtime/arrow_realtime_store_factory.h   |  1 -
 include/paimon/realtime/realtime_store.h      | 39 ++++++++-----------
 src/paimon/core/mergetree/merge_tree_writer.h |  5 +--
 .../realtime/primary_key_realtime_store.h     |  2 +-
 .../realtime/realtime_primary_key_writer.h    |  1 -
 5 files changed, 19 insertions(+), 29 deletions(-)

diff --git a/include/paimon/realtime/arrow_realtime_store_factory.h b/include/paimon/realtime/arrow_realtime_store_factory.h
index da1b8de36..153d524d4 100644
--- a/include/paimon/realtime/arrow_realtime_store_factory.h
+++ b/include/paimon/realtime/arrow_realtime_store_factory.h
@@ -26,7 +26,6 @@ namespace paimon {
 /// Factory for Paimon's default Arrow-backed `RealtimeStore`.
 class PAIMON_EXPORT ArrowRealtimeStoreFactory : public RealtimeStoreFactory {
  public:
-    /// Creates the built-in append or in-memory primary-key store.
     Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override;
 };
 
diff --git a/include/paimon/realtime/realtime_store.h b/include/paimon/realtime/realtime_store.h
index 90c6ce0a8..60d1afc39 100644
--- a/include/paimon/realtime/realtime_store.h
+++ b/include/paimon/realtime/realtime_store.h
@@ -69,10 +69,10 @@ struct PAIMON_EXPORT RealtimeStoreCreateRequest {
 
 /// A record batch and its framework-assigned contiguous offset range.
 ///
-/// Append-mode batches contain table write fields, and row `i` is associated with
-/// `offset_range.begin + i`. Primary-key batches contain the prepared transport schema supplied
-/// to the factory and are physically sorted by full primary key then sequence number; their
-/// per-row `_REALTIME_OFFSET` field preserves the original write-order offset after sorting.
+/// Append-mode batches contain table write fields, and row `i` has offset
+/// `offset_range.begin + i`. Primary-key batches use the prepared transport schema, are sorted
+/// by full primary key then sequence number, and retain the original offset in
+/// `_REALTIME_OFFSET`.
 struct PAIMON_EXPORT RealtimeWriteBatch {
     /// Input batch whose ownership is transferred to `RealtimeStore::Write`.
     std::unique_ptr<RecordBatch> batch;
@@ -147,14 +147,10 @@ class PAIMON_EXPORT RealtimeStore {
 
     /// Creates readers that expose all rows in a sealed segment for Paimon file writing.
     ///
-    /// Concatenating the returned readers must produce every sealed row exactly once. Append-mode
-    /// readers preserve write order and contain `_VALUE_KIND` followed by table write fields.
-    /// Primary-key readers expose raw prepared rows. Each returned reader's complete stream,
-    /// including across `NextBatch` boundaries, is sorted by full primary key then sequence
-    /// number; all readers collectively cover sealed mutations exactly once. Reader cardinality is
-    /// independent of the number of writes. Paimon adapts and merges those rows before writing
-    /// files. Paimon validates the complete ordering and coverage before publishing generated file
-    /// state; a violation fails the prepare operation.
+    /// The returned readers collectively expose every sealed row exactly once. Append-mode readers
+    /// preserve write order and contain `_VALUE_KIND` followed by table write fields. Primary-key
+    /// readers use the prepared transport schema; each reader's complete stream is sorted by full
+    /// primary key then sequence number.
     virtual Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
         const std::shared_ptr<RealtimeSegmentHandle>& segment) = 0;
 
@@ -164,18 +160,15 @@ class PAIMON_EXPORT RealtimeStore {
     /// also provide a consistent snapshot when a write or seal is in progress.
     virtual Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() = 0;
 
-    /// Creates readers over rows in `view`. Append mode returns rows whose offsets are greater
-    /// than or equal to `offset_begin`. Primary-key mode ignores `offset_begin` and returns raw
-    /// prepared rows; Paimon applies offset filtering, projection, and merge-on-read adaptation.
+    /// Creates readers over rows in `view`. Append mode returns rows whose offsets are greater than
+    /// or equal to `offset_begin`; primary-key mode ignores `offset_begin`.
     ///
-    /// Append-mode output batches contain `_VALUE_KIND` first, followed by requested fields except
-    /// a duplicate `_VALUE_KIND`; all returned append readers collectively cover every matching
-    /// row exactly once. Primary-key output batches use the prepared transport schema and may
-    /// contain multiple mutations per key. Each returned primary-key reader's complete stream is
-    /// sorted by full primary key then sequence number, and all readers collectively cover raw
-    /// mutations exactly once. Reader cardinality is independent of the number of writes. Paimon
-    /// validates ordering while adapting each complete reader stream and retains `view` for the
-    /// lifetime of the resulting framework reader.
+    /// Append-mode batches contain `_VALUE_KIND` followed by the requested fields except a duplicate
+    /// `_VALUE_KIND`, and collectively expose every matching row exactly once. Primary-key batches
+    /// use the prepared transport schema and may contain multiple mutations per key; each reader's
+    /// complete stream is sorted by full primary key then sequence number, and the readers
+    /// collectively expose every raw mutation exactly once. Paimon retains `view` for the lifetime
+    /// of the resulting framework reader.
     virtual Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
         const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
         const RealtimeQueryContext& context) = 0;
diff --git a/src/paimon/core/mergetree/merge_tree_writer.h b/src/paimon/core/mergetree/merge_tree_writer.h
index cea07f3e4..01efd975c 100644
--- a/src/paimon/core/mergetree/merge_tree_writer.h
+++ b/src/paimon/core/mergetree/merge_tree_writer.h
@@ -70,9 +70,8 @@ class MergeTreeWriter : public BatchWriter {
 
     Status Write(std::unique_ptr<RecordBatch>&& batch) override;
 
-    /// Consumes readers whose complete streams are individually sorted by primary key and
-    /// sequence number. Readers are closed on success or failure; an error may leave generated
-    /// file state unpublished, so the caller must discard this writer and replay its input.
+    /// Consumes readers whose complete streams are individually sorted by primary key and sequence
+    /// number. Readers are closed on success or failure.
     Status WriteSortedReaders(std::vector<std::unique_ptr<KeyValueRecordReader>>&& readers);
 
     Status Compact(bool full_compaction) override;
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.h b/src/paimon/core/realtime/primary_key_realtime_store.h
index d6a23ccf9..f779b4d7d 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.h
+++ b/src/paimon/core/realtime/primary_key_realtime_store.h
@@ -35,7 +35,7 @@ class TableSchema;
 
 Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options, const TableSchema& schema);
 
-/// In-memory store for prepared primary-key real-time batches.
+/// Internal in-memory implementation of the default primary-key `RealtimeStore`.
 class PrimaryKeyRealtimeStore final : public RealtimeStore {
  public:
     static Result<std::shared_ptr<PrimaryKeyRealtimeStore>> Create(
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.h b/src/paimon/core/realtime/realtime_primary_key_writer.h
index 2eaf7ce24..d65c7e533 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.h
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.h
@@ -42,7 +42,6 @@ class FieldsComparator;
 class RealtimeContextImpl;
 struct RealtimeStoreState;
 
-/// Coordinates framework-prepared primary-key real-time writes.
 class RealtimePrimaryKeyWriter final : public BatchWriter {
  public:
     static Result<std::shared_ptr<RealtimePrimaryKeyWriter>> Create(

From ff0c7bdbef7f50eaf79f26d9645202735868d802 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Tue, 25 Aug 2026 00:32:24 +0800
Subject: [PATCH 24/62] fix(realtime): strengthen primary-key recovery coverage

---
 .../core/mergetree/merge_tree_writer_test.cpp |  58 +++++++
 .../core/realtime/realtime_context_impl.cpp   |  16 +-
 .../core/realtime/realtime_context_test.cpp   |   6 +-
 .../table/source/key_value_table_read.cpp     |   3 +
 test/inte/realtime_write_inte_test.cpp        | 158 ++++++++++++++++++
 5 files changed, 237 insertions(+), 4 deletions(-)

diff --git a/src/paimon/core/mergetree/merge_tree_writer_test.cpp b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
index 63e896573..9ce5498cb 100644
--- a/src/paimon/core/mergetree/merge_tree_writer_test.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
@@ -530,6 +530,64 @@ TEST_P(MergeTreeWriterTest, TestSortedReaders) {
     ASSERT_EQ(1, new_file->delete_row_count);
 }
 
+TEST_P(MergeTreeWriterTest, TestMergeSortedReaders) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options,
+                         CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}}));
+
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    auto path_factory = std::make_shared<DataFilePathFactory>();
+    ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr));
+    ASSERT_OK_AND_ASSIGN(auto merge_writer,
+                         CreateMergeWriter(/*last_sequence_number=*/4, dir->Str(), path_factory,
+                                           /*schema_id=*/7, options));
+
+    auto first_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(write_type_, R"([
+      [5, 0, "Alice", 10, 0, 15.1],
+      [7, 0, "Carol", 20, 1, 17.1],
+      [10, 0, "Eve", 30, 2, 20.1]
+    ])")
+            .ValueOrDie());
+    auto second_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(write_type_, R"([
+      [6, 0, "Bob", 11, 0, 16.1],
+      [8, 3, "Carol", 21, 1, null],
+      [9, 0, "David", 22, 2, 19.1]
+    ])")
+            .ValueOrDie());
+    bool first_closed = false;
+    bool second_closed = false;
+    std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
+    sorted_readers.push_back(std::make_unique<TrackingKeyValueRecordReader>(
+        CreateSingleReader(first_array), &first_closed));
+    sorted_readers.push_back(std::make_unique<TrackingKeyValueRecordReader>(
+        CreateSingleReader(second_array), &second_closed));
+
+    ASSERT_OK(merge_writer->WriteSortedReaders(std::move(sorted_readers)));
+    ASSERT_TRUE(first_closed);
+    ASSERT_TRUE(second_closed);
+    ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment, merge_writer->PrepareCommit(false));
+    ASSERT_OK(merge_writer->Close());
+
+    ASSERT_EQ(1, commit_increment.GetNewFilesIncrement().NewFiles().size());
+    const std::shared_ptr<DataFileMeta>& new_file =
+        commit_increment.GetNewFilesIncrement().NewFiles()[0];
+    ASSERT_EQ(5, new_file->row_count);
+    ASSERT_EQ(1, new_file->delete_row_count);
+    std::shared_ptr<arrow::ChunkedArray> expected_array;
+    ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(write_type_, {R"([
+      [5, 0, "Alice", 10, 0, 15.1],
+      [6, 0, "Bob", 11, 0, 16.1],
+      [8, 3, "Carol", 21, 1, null],
+      [9, 0, "David", 22, 2, 19.1],
+      [10, 0, "Eve", 30, 2, 20.1]
+    ])"},
+                                                                 &expected_array)
+                    .ok());
+    CheckFileContent(path_factory->ToPath(new_file), expected_array);
+}
+
 TEST_P(MergeTreeWriterTest, TestSortedReaderOwnership) {
     ASSERT_OK_AND_ASSIGN(CoreOptions options,
                          CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}}));
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index ba4c8b7a6..736ebb02d 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -59,6 +59,17 @@ bool SameMode(const RealtimeStoreCreateConfig& left, const RealtimeStoreCreateCo
     return true;
 }
 
+std::string PartitionToString(const std::map<std::string, std::string>& partition) {
+    std::string result = "{";
+    for (auto iter = partition.begin(); iter != partition.end(); ++iter) {
+        if (iter != partition.begin()) {
+            result += ", ";
+        }
+        result += iter->first + "=" + iter->second;
+    }
+    return result + "}";
+}
+
 }  // namespace
 
 RealtimeContextImpl::RealtimeContextImpl(const std::shared_ptr<RealtimeStoreFactory>& factory)
@@ -120,8 +131,9 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     if (iter != stores_.end()) {
         if (!SameMode(iter->second.mode_config, request.mode_config) ||
             !iter->second.write_schema->Equals(*requested_schema, /*check_metadata=*/true)) {
-            return Status::Invalid(
-                "real-time store schema or mode does not match the registered store");
+            return Status::Invalid("real-time store schema or mode mismatch for partition " +
+                                   PartitionToString(key.partition) + ", bucket " +
+                                   std::to_string(key.bucket) + "; recreate the RealtimeContext");
         }
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeReadView> read_view,
                                iter->second.store->AcquireReadView());
diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp
index 2b47e9dc9..916b46aad 100644
--- a/src/paimon/core/realtime/realtime_context_test.cpp
+++ b/src/paimon/core/realtime/realtime_context_test.cpp
@@ -171,13 +171,15 @@ TEST(RealtimeContextTest, TestRejectsMismatchedSchemaOnStoreReuse) {
     ASSERT_NOK_WITH_MSG(
         GetOrCreateAppendStore(context, partition, 0, MakeWriteSchema(arrow::int32(), metadata), {},
                                GetDefaultPool()),
-        "schema or mode does not match");
+        "schema or mode mismatch for partition {dt=2026-08-02}, bucket 0; recreate the "
+        "RealtimeContext");
     ASSERT_NOK_WITH_MSG(
         GetOrCreateAppendStore(
             context, partition, 0,
             MakeWriteSchema(arrow::int64(), arrow::key_value_metadata({"identity"}, {"v2"})), {},
             GetDefaultPool()),
-        "schema or mode does not match");
+        "schema or mode mismatch for partition {dt=2026-08-02}, bucket 0; recreate the "
+        "RealtimeContext");
     ASSERT_EQ(1, factory->stores.size());
 }
 
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index 96f3f00d9..3532b59b4 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -253,6 +253,9 @@ Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateRealtimeReader(
     if (realtime_split->Version() != RealtimeSplit::kCurrentVersion) {
         return Status::Invalid("unsupported real-time split version");
     }
+    if (realtime_split->MemoryEndOffset() < realtime_split->CommittedEndOffset()) {
+        return Status::Invalid("real-time split memory end offset precedes committed end offset");
+    }
     const std::shared_ptr<RealtimeContext> realtime_context = context_->GetRealtimeContext();
     if (!realtime_context) {
         return Status::Invalid("reading a real-time split requires a real-time context");
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index a393f838d..53737fc2b 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -1281,6 +1281,30 @@ class RealtimeWriteInteTest : public ::testing::Test {
         ASSERT_EQ(expected_rows, actual_rows);
     }
 
+    void ReplayPkWalAndCommit(const std::vector<Row>& wal,
+                              const std::vector<RecordBatch::RowKind>& row_kinds,
+                              int64_t commit_identifier,
+                              const std::vector<Row>& expected_rows) const {
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                             RealtimeContext::Create());
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                             CreateRealtimeWriter(realtime_context));
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                             MakeBatch(wal, /*partitioned=*/false, /*bucket=*/0, row_kinds));
+        ASSERT_OK(writer->Write(std::move(batch)));
+        ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+                             writer->PrepareCommitWithProgress(commit_identifier));
+        ASSERT_EQ(1, progress.size());
+        ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, commit_identifier));
+        ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
+        ASSERT_OK(writer->Close());
+        writer.reset();
+        realtime_context.reset();
+
+        ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows());
+        ASSERT_EQ(expected_rows, actual_rows);
+    }
+
     void CheckDropDatePartitionRemovesOffset(bool legacy_partition_name_enabled) {
         fields_ = {arrow::field("id", arrow::int64()), arrow::field("payload", arrow::utf8()),
                    arrow::field("pt", arrow::date32())};
@@ -2861,6 +2885,44 @@ TEST_F(RealtimeWriteInteTest, TestPlanExcludesRowsWrittenAfterMemoryEndOffset) {
     ASSERT_OK(writer->Close());
 }
 
+TEST_F(RealtimeWriteInteTest, TestPkRejectsReversedVisibleOffsets) {
+    CreatePkTable();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+    std::vector<Row> rows = {{1, "one", "p0"}, {2, "two", "p0"}};
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         MakeBatch(rows, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(batch)));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
+                         CreatePlan(realtime_context, /*predicate=*/nullptr));
+    ASSERT_EQ(1, plan->Splits().size());
+    std::shared_ptr<RealtimeSplit> split =
+        std::dynamic_pointer_cast<RealtimeSplit>(plan->Splits()[0]);
+    ASSERT_NE(nullptr, split);
+    std::vector<std::shared_ptr<Split>> disk_splits = split->DiskSplits();
+    std::vector<std::shared_ptr<Split>> invalid_splits = {std::make_shared<RealtimeSplit>(
+        split->Version(), split->SnapshotId(), split->Partition(), split->Bucket(),
+        std::move(disk_splits), split->MemoryEndOffset() + 1, split->MemoryEndOffset(),
+        split->OpaqueTicket())};
+
+    ReadContextBuilder read_builder(table_path_);
+    read_builder.SetOptions(options_)
+        .SetReadFieldNames({"id", "payload", "pt"})
+        .WithRealtimeContext(realtime_context)
+        .WithMemoryPool(pool_);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableRead> table_read,
+                         TableRead::Create(std::move(read_context)));
+    ASSERT_NOK_WITH_MSG(table_read->CreateReader(invalid_splits),
+                        "memory end offset precedes committed end offset");
+
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(plan, realtime_context));
+    ASSERT_EQ(rows, actual_rows);
+    ASSERT_OK(writer->Close());
+}
+
 TEST_F(RealtimeWriteInteTest, TestReadFailsAfterRealtimeSplitTicketExpires) {
     options_[Options::REALTIME_READ_VIEW_TTL] = "10 ms";
     CreateTable(/*partition_keys=*/{});
@@ -4392,4 +4454,100 @@ TEST_F(RealtimeWriteInteTest, TestRestoreOffsetFromCommittedSnapshot) {
     ASSERT_EQ(5, second_committed_offsets.at(partition_bucket));
 }
 
+TEST_F(RealtimeWriteInteTest, TestPkExternalCommitRecovery) {
+    CreatePkTable();
+    const std::vector<Row> seed_rows = {{99, "seed", "p0"}};
+    ReplayPkWalAndCommit(seed_rows, /*row_kinds=*/{}, /*commit_identifier=*/0, seed_rows);
+
+    const std::vector<Row> wal = {
+        {1, "one", "p0"}, {1, "one-new", "p0"}, {2, "deleted", "p0"}, {3, "three", "p0"}};
+    const std::vector<RecordBatch::RowKind> row_kinds = {
+        RecordBatch::RowKind::INSERT, RecordBatch::RowKind::UPDATE_AFTER,
+        RecordBatch::RowKind::DELETE, RecordBatch::RowKind::INSERT};
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> failed_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> failed_writer,
+                         CreateRealtimeWriter(failed_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> failed_batch,
+                         MakeBatch(wal, /*partitioned=*/false, /*bucket=*/0, row_kinds));
+    ASSERT_OK(failed_writer->Write(std::move(failed_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> failed_progress,
+                         failed_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_EQ(1, failed_progress.size());
+    CommitContextBuilder commit_builder(table_path_, commit_user_);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
+                         commit_builder.SetOptions(options_).Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> commit,
+                         FileStoreCommit::Create(std::move(commit_context)));
+    IOHook* io_hook = IOHook::GetInstance();
+    ScopeGuard hook_guard([io_hook]() { io_hook->Clear(); });
+    io_hook->Reset(/*pos=*/0, IOHook::Mode::RETURN_ERROR);
+    Result<int64_t> failed_commit =
+        commit->CommitWithProgress(failed_progress, /*commit_identifier=*/1,
+                                   /*watermark=*/std::nullopt);
+    io_hook->Clear();
+    ASSERT_TRUE(failed_commit.status().IsIOError()) << failed_commit.status().ToString();
+    ASSERT_OK(failed_writer->Close());
+    failed_writer.reset();
+    failed_context.reset();
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_failure, ReadRows());
+    ASSERT_EQ(seed_rows, rows_after_failure);
+
+    const std::vector<Row> expected_rows = {
+        {1, "one-new", "p0"}, {3, "three", "p0"}, {99, "seed", "p0"}};
+    ReplayPkWalAndCommit(wal, row_kinds, /*commit_identifier=*/1, expected_rows);
+}
+
+TEST_F(RealtimeWriteInteTest, TestPkRefreshRecovery) {
+    CreatePkTable();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> failed_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> failed_writer,
+                         CreateRealtimeWriter(failed_context));
+
+    const std::vector<Row> base_rows = {{1, "one", "p0"}, {2, "two", "p0"}};
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> base_batch,
+                         MakeBatch(base_rows, /*partitioned=*/false));
+    ASSERT_OK(failed_writer->Write(std::move(base_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> base_progress,
+                         failed_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_OK_AND_ASSIGN(int64_t base_snapshot_id, Commit(base_progress, /*commit_identifier=*/0));
+    ASSERT_OK(failed_writer->RefreshCommittedSnapshot(base_snapshot_id));
+
+    const std::vector<Row> committed_wal = {
+        {1, "one-new", "p0"}, {2, "deleted", "p0"}, {3, "three", "p0"}};
+    const std::vector<RecordBatch::RowKind> committed_kinds = {RecordBatch::RowKind::UPDATE_AFTER,
+                                                               RecordBatch::RowKind::DELETE,
+                                                               RecordBatch::RowKind::INSERT};
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<RecordBatch> committed_batch,
+        MakeBatch(committed_wal, /*partitioned=*/false, /*bucket=*/0, committed_kinds));
+    ASSERT_OK(failed_writer->Write(std::move(committed_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> committed_progress,
+                         failed_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_OK_AND_ASSIGN(int64_t committed_snapshot_id,
+                         Commit(committed_progress, /*commit_identifier=*/1));
+
+    const std::vector<Row> replay_wal = {{4, "four", "p0"}};
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> replay_batch,
+                         MakeBatch(replay_wal, /*partitioned=*/false));
+    ASSERT_OK(failed_writer->Write(std::move(replay_batch)));
+    IOHook* io_hook = IOHook::GetInstance();
+    ScopeGuard hook_guard([io_hook]() { io_hook->Clear(); });
+    io_hook->Reset(/*pos=*/0, IOHook::Mode::RETURN_ERROR);
+    Status failed_refresh = failed_writer->RefreshCommittedSnapshot(committed_snapshot_id);
+    io_hook->Clear();
+    ASSERT_TRUE(failed_refresh.IsIOError()) << failed_refresh.ToString();
+    ASSERT_OK(failed_writer->Close());
+    failed_writer.reset();
+    failed_context.reset();
+    const std::vector<Row> committed_rows = {{1, "one-new", "p0"}, {3, "three", "p0"}};
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_failure, ReadRows());
+    ASSERT_EQ(committed_rows, rows_after_failure);
+
+    const std::vector<Row> expected_rows = {
+        {1, "one-new", "p0"}, {3, "three", "p0"}, {4, "four", "p0"}};
+    ReplayPkWalAndCommit(replay_wal, /*row_kinds=*/{}, /*commit_identifier=*/2, expected_rows);
+}
+
 }  // namespace paimon::test

From 83ff3cbf98fee672910fed7228648ade0fef5e2b Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Tue, 25 Aug 2026 10:15:22 +0800
Subject: [PATCH 25/62] test(realtime): strengthen failure recovery coverage

---
 include/paimon/realtime/realtime_store.h      |  12 +-
 .../core/io/single_file_writer_test.cpp       |   4 +-
 .../realtime/primary_key_realtime_store.cpp   |   8 +-
 .../realtime/primary_key_realtime_store.h     |   2 +-
 .../table/source/key_value_table_read.cpp     |   5 +-
 test/inte/realtime_write_inte_test.cpp        | 124 ++++++++++++++++++
 6 files changed, 141 insertions(+), 14 deletions(-)

diff --git a/include/paimon/realtime/realtime_store.h b/include/paimon/realtime/realtime_store.h
index 60d1afc39..03ef279a3 100644
--- a/include/paimon/realtime/realtime_store.h
+++ b/include/paimon/realtime/realtime_store.h
@@ -163,12 +163,12 @@ class PAIMON_EXPORT RealtimeStore {
     /// Creates readers over rows in `view`. Append mode returns rows whose offsets are greater than
     /// or equal to `offset_begin`; primary-key mode ignores `offset_begin`.
     ///
-    /// Append-mode batches contain `_VALUE_KIND` followed by the requested fields except a duplicate
-    /// `_VALUE_KIND`, and collectively expose every matching row exactly once. Primary-key batches
-    /// use the prepared transport schema and may contain multiple mutations per key; each reader's
-    /// complete stream is sorted by full primary key then sequence number, and the readers
-    /// collectively expose every raw mutation exactly once. Paimon retains `view` for the lifetime
-    /// of the resulting framework reader.
+    /// Append-mode batches contain `_VALUE_KIND` followed by the requested fields except a
+    /// duplicate `_VALUE_KIND`, and collectively expose every matching row exactly once.
+    /// Primary-key batches use the prepared transport schema and may contain multiple mutations
+    /// per key; each reader's complete stream is sorted by full primary key then sequence number,
+    /// and the readers collectively expose every raw mutation exactly once. Paimon retains `view`
+    /// for the lifetime of the resulting framework reader.
     virtual Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
         const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
         const RealtimeQueryContext& context) = 0;
diff --git a/src/paimon/core/io/single_file_writer_test.cpp b/src/paimon/core/io/single_file_writer_test.cpp
index 78fce54c7..4136702e8 100644
--- a/src/paimon/core/io/single_file_writer_test.cpp
+++ b/src/paimon/core/io/single_file_writer_test.cpp
@@ -18,8 +18,10 @@
 
 #include "paimon/core/io/single_file_writer.h"
 
+#include <functional>
 #include <map>
-#include <utility>
+#include <memory>
+#include <string>
 
 #include "arrow/api.h"
 #include "arrow/c/abi.h"
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index e4c480377..22fab1b08 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -418,9 +418,9 @@ class PrimaryKeyRealtimeStore::Impl {
         return readers;
     }
 
-    Status AdvanceCommittedOffset(int64_t committed_end) {
+    Status AdvanceCommittedOffset(int64_t committed_end_offset) {
         std::lock_guard<std::mutex> lock(mutex_);
-        while (!sealed_.empty() && sealed_.front()->GetOffsetRange().end <= committed_end) {
+        while (!sealed_.empty() && sealed_.front()->GetOffsetRange().end <= committed_end_offset) {
             sealed_.erase(sealed_.begin());
         }
         return Status::OK();
@@ -499,8 +499,8 @@ Result<std::vector<std::unique_ptr<BatchReader>>> PrimaryKeyRealtimeStore::Creat
     const RealtimeQueryContext& context) {
     return impl_->CreateQueryReaders(view, offset, context);
 }
-Status PrimaryKeyRealtimeStore::AdvanceCommittedOffset(int64_t offset) {
-    return impl_->AdvanceCommittedOffset(offset);
+Status PrimaryKeyRealtimeStore::AdvanceCommittedOffset(int64_t committed_end_offset) {
+    return impl_->AdvanceCommittedOffset(committed_end_offset);
 }
 uint64_t PrimaryKeyRealtimeStore::GetMemoryUsage() const {
     return impl_->GetMemoryUsage();
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.h b/src/paimon/core/realtime/primary_key_realtime_store.h
index f779b4d7d..52f9a6076 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.h
+++ b/src/paimon/core/realtime/primary_key_realtime_store.h
@@ -53,7 +53,7 @@ class PrimaryKeyRealtimeStore final : public RealtimeStore {
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
         const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
         const RealtimeQueryContext& context) override;
-    Status AdvanceCommittedOffset(int64_t committed_offset) override;
+    Status AdvanceCommittedOffset(int64_t committed_end_offset) override;
     uint64_t GetMemoryUsage() const override;
 
  private:
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index 3532b59b4..af585a1bb 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -150,7 +150,7 @@ Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateReader(
     const std::shared_ptr<Split>& split) {
     std::shared_ptr<RealtimeSplit> realtime_split = std::dynamic_pointer_cast<RealtimeSplit>(split);
     if (realtime_split) {
-        return CreateRealtimeReader(realtime_split, true);
+        return CreateRealtimeReader(realtime_split, /*release_ticket=*/true);
     }
 
     std::shared_ptr<Split> dispatch_split = split;
@@ -221,7 +221,8 @@ Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateReader(
             std::dynamic_pointer_cast<RealtimeSplit>(split);
         if (realtime_split) {
             PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> reader,
-                                   CreateRealtimeReader(realtime_split, false));
+                                   CreateRealtimeReader(realtime_split,
+                                                        /*release_ticket=*/false));
             readers.push_back(std::move(reader));
             realtime_splits.push_back(std::move(realtime_split));
         } else {
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index 53737fc2b..918d1dd35 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -28,6 +28,7 @@
 #include <map>
 #include <memory>
 #include <mutex>
+#include <new>
 #include <set>
 #include <string>
 #include <thread>
@@ -78,6 +79,61 @@
 namespace paimon::test {
 namespace {
 
+class FailAllocationMemoryPool final : public MemoryPool {
+ public:
+    explicit FailAllocationMemoryPool(const std::shared_ptr<MemoryPool>& delegate)
+        : delegate_(delegate) {}
+
+    void FailAfterAllocations(int64_t successful_allocations) {
+        allocations_before_failure_.store(successful_allocations, std::memory_order_release);
+    }
+
+    void* Malloc(uint64_t size, uint64_t alignment = 0) override {
+        if (ShouldFail()) {
+            throw std::bad_alloc();
+        }
+        return delegate_->Malloc(size, alignment);
+    }
+
+    void* Realloc(void* p, size_t old_size, size_t new_size, uint64_t alignment = 0) override {
+        if (ShouldFail()) {
+            throw std::bad_alloc();
+        }
+        return delegate_->Realloc(p, old_size, new_size, alignment);
+    }
+
+    void Free(void* p, uint64_t size) override {
+        delegate_->Free(p, size);
+    }
+
+    void Free(void* p, uint64_t size, uint64_t alignment) override {
+        delegate_->Free(p, size, alignment);
+    }
+
+    uint64_t CurrentUsage() const override {
+        return delegate_->CurrentUsage();
+    }
+
+    uint64_t MaxMemoryUsage() const override {
+        return delegate_->MaxMemoryUsage();
+    }
+
+ private:
+    bool ShouldFail() {
+        int64_t remaining = allocations_before_failure_.load(std::memory_order_acquire);
+        while (remaining >= 0) {
+            if (allocations_before_failure_.compare_exchange_weak(remaining, remaining - 1,
+                                                                  std::memory_order_acq_rel)) {
+                return remaining == 0;
+            }
+        }
+        return false;
+    }
+
+    std::shared_ptr<MemoryPool> delegate_;
+    std::atomic<int64_t> allocations_before_failure_{-1};
+};
+
 class TrackingRealtimeReadView final : public RealtimeReadView {
  public:
     explicit TrackingRealtimeReadView(std::shared_ptr<RealtimeReadView> delegate)
@@ -2106,6 +2162,7 @@ TEST_F(RealtimeWriteInteTest, TestPkRecovery) {
     ASSERT_EQ(1, NewFiles(progress).size());
     ASSERT_EQ(2, NewFiles(progress)[0]->min_sequence_number);
     ASSERT_EQ(memory_sequences.back(), NewFiles(progress)[0]->max_sequence_number);
+    ASSERT_EQ(1, NewFiles(progress)[0]->delete_row_count);
     ASSERT_OK(Commit(progress, /*commit_identifier=*/1));
     ASSERT_OK(first_writer->Close());
     first_context.reset();
@@ -4454,6 +4511,73 @@ TEST_F(RealtimeWriteInteTest, TestRestoreOffsetFromCommittedSnapshot) {
     ASSERT_EQ(5, second_committed_offsets.at(partition_bucket));
 }
 
+TEST_F(RealtimeWriteInteTest, TestPkWriteFailureRecovery) {
+    CreatePkTable();
+    const std::vector<Row> seed_rows = {{99, "seed", "p0"}};
+    ReplayPkWalAndCommit(seed_rows, /*row_kinds=*/{}, /*commit_identifier=*/0, seed_rows);
+
+    const std::vector<Row> wal = {
+        {1, "one", "p0"}, {1, "one-new", "p0"}, {2, "deleted", "p0"}, {3, "three", "p0"}};
+    const std::vector<RecordBatch::RowKind> row_kinds = {
+        RecordBatch::RowKind::INSERT, RecordBatch::RowKind::UPDATE_AFTER,
+        RecordBatch::RowKind::DELETE, RecordBatch::RowKind::INSERT};
+    std::shared_ptr<FailAllocationMemoryPool> failing_pool =
+        std::make_shared<FailAllocationMemoryPool>(pool_);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> failed_context,
+                         RealtimeContext::Create());
+    WriteContextBuilder failed_builder(table_path_, commit_user_);
+    failed_builder.SetOptions(options_)
+        .WithStreamingMode(true)
+        .WithRealtimeContext(failed_context)
+        .WithMemoryPool(failing_pool);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> failed_write_context,
+                         failed_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> failed_writer,
+                         FileStoreWrite::Create(std::move(failed_write_context)));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> empty_batch,
+                         MakeUnpartitionedBatchFromJson("[]"));
+    ASSERT_OK(failed_writer->Write(std::move(empty_batch)));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> failed_batch,
+                         MakeBatch(wal, /*partitioned=*/false, /*bucket=*/0, row_kinds));
+    failing_pool->FailAfterAllocations(1);
+    Status failed_write = failed_writer->Write(std::move(failed_batch));
+    ASSERT_TRUE(failed_write.IsOutOfMemory()) << failed_write.ToString();
+    ASSERT_OK(failed_writer->Close());
+    failed_writer.reset();
+    failed_context.reset();
+
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_failure, ReadRows());
+    ASSERT_EQ(seed_rows, rows_after_failure);
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> replay_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> replay_writer,
+                         CreateRealtimeWriter(replay_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> replay_batch,
+                         MakeBatch(wal, /*partitioned=*/false, /*bucket=*/0, row_kinds));
+    ASSERT_OK(replay_writer->Write(std::move(replay_batch)));
+    const std::vector<Row> expected_rows = {
+        {1, "one-new", "p0"}, {3, "three", "p0"}, {99, "seed", "p0"}};
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> replayed_rows, ReadRows(replay_context));
+    ASSERT_EQ(expected_rows, replayed_rows);
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+                         replay_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_EQ(1, progress.size());
+    ASSERT_EQ(OffsetRange(1, 5), progress[0].offset_range);
+    ASSERT_EQ(1, NewFiles(progress).size());
+    ASSERT_EQ(1, NewFiles(progress)[0]->delete_row_count);
+    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, /*commit_identifier=*/1));
+    ASSERT_OK(replay_writer->RefreshCommittedSnapshot(snapshot_id));
+    ASSERT_OK(replay_writer->Close());
+    replay_writer.reset();
+    replay_context.reset();
+
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> persisted_rows, ReadRows());
+    ASSERT_EQ(expected_rows, persisted_rows);
+    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap committed_offsets, ReadCommittedOffsets());
+    ASSERT_EQ(5, committed_offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
+}
+
 TEST_F(RealtimeWriteInteTest, TestPkExternalCommitRecovery) {
     CreatePkTable();
     const std::vector<Row> seed_rows = {{99, "seed", "p0"}};

From 55581822a7d8ff87177711719a5f870431f5c719 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Tue, 25 Aug 2026 11:24:04 +0800
Subject: [PATCH 26/62] refactor(realtime): simplify primary key state and
 tests

---
 src/paimon/common/table/special_fields.h      |   7 +
 .../common/table/special_fields_test.cpp      |   8 +
 .../merged_key_value_record_reader_test.cpp   |  12 +-
 .../operation/key_value_file_store_write.cpp  |   7 +-
 .../realtime/prepared_key_value_reader.cpp    |   3 +-
 .../primary_key_realtime_store_test.cpp       |   5 +-
 .../core/realtime/realtime_context_impl.cpp   |  12 +-
 .../core/realtime/realtime_context_impl.h     |  16 +-
 .../core/realtime/realtime_context_test.cpp   |  58 +++
 src/paimon/core/realtime/realtime_fields.h    |  35 --
 .../realtime/realtime_primary_key_writer.cpp  |   3 +-
 .../table/source/append_only_table_read.cpp   |   5 +-
 .../table/source/key_value_table_read.cpp     |   8 +-
 test/inte/realtime_write_inte_test.cpp        | 401 ++++++------------
 14 files changed, 234 insertions(+), 346 deletions(-)
 delete mode 100644 src/paimon/core/realtime/realtime_fields.h

diff --git a/src/paimon/common/table/special_fields.h b/src/paimon/common/table/special_fields.h
index 74b95b19c..3279bfed6 100644
--- a/src/paimon/common/table/special_fields.h
+++ b/src/paimon/common/table/special_fields.h
@@ -66,6 +66,13 @@ struct SpecialFields {
         return data_field;
     }
 
+    static const DataField& RealtimeOffset() {
+        static const DataField data_field =
+            DataField(SpecialFieldIds::REALTIME_OFFSET,
+                      arrow::field("_REALTIME_OFFSET", arrow::int64(), false));
+        return data_field;
+    }
+
     static bool IsSystemField(const std::string& field_name) {
         if (StringUtils::StartsWith(field_name, KEY_FIELD_PREFIX)) {
             return true;
diff --git a/src/paimon/common/table/special_fields_test.cpp b/src/paimon/common/table/special_fields_test.cpp
index 68e805fd6..b61d289b0 100644
--- a/src/paimon/common/table/special_fields_test.cpp
+++ b/src/paimon/common/table/special_fields_test.cpp
@@ -55,6 +55,13 @@ TEST(SpecialFieldsTest, TestIndexScore) {
     ASSERT_EQ(SpecialFields::IndexScore().Type()->id(), arrow::Type::FLOAT);
 }
 
+TEST(SpecialFieldsTest, TestRealtimeOffset) {
+    ASSERT_EQ(SpecialFields::RealtimeOffset().Id(), SpecialFieldIds::REALTIME_OFFSET);
+    ASSERT_EQ(SpecialFields::RealtimeOffset().Name(), "_REALTIME_OFFSET");
+    ASSERT_EQ(SpecialFields::RealtimeOffset().Type()->id(), arrow::Type::INT64);
+    ASSERT_FALSE(SpecialFields::RealtimeOffset().Nullable());
+}
+
 TEST(SpecialFieldsTest, TestKeyValueSpecialFieldCount) {
     ASSERT_EQ(SpecialFields::KEY_VALUE_SPECIAL_FIELD_COUNT, 2);
 }
@@ -66,6 +73,7 @@ TEST(SpecialFieldsTest, TestIsSystemField) {
     ASSERT_TRUE(SpecialFields::IsSystemField("rowkind"));
     ASSERT_TRUE(SpecialFields::IsSystemField("_ROW_ID"));
     ASSERT_TRUE(SpecialFields::IsSystemField("_INDEX_SCORE"));
+    ASSERT_FALSE(SpecialFields::IsSystemField("_REALTIME_OFFSET"));
     ASSERT_TRUE(SpecialFields::IsSystemField("_KEY_0"));
 }
 
diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index a0d65205c..79217828c 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -36,7 +36,6 @@
 #include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
 #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
 #include "paimon/core/realtime/prepared_key_value_reader.h"
-#include "paimon/core/realtime/realtime_fields.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/realtime/offset_range.h"
 #include "paimon/testing/mock/mock_file_batch_reader.h"
@@ -44,7 +43,6 @@
 #include "paimon/testing/utils/key_value_checker.h"
 #include "paimon/testing/utils/read_result_collector.h"
 #include "paimon/testing/utils/testharness.h"
-#include "paimon/utils/special_field_ids.h"
 
 namespace paimon::test {
 
@@ -62,7 +60,7 @@ std::shared_ptr<arrow::Schema> MakePreparedSchema(const arrow::FieldVector& valu
         DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
         DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
             ->WithNullable(false),
-        DataField::ConvertDataFieldToArrowField(RealtimeOffsetField())->WithNullable(false)};
+        DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset())};
     prepared_fields.insert(prepared_fields.end(), value_fields.begin(), value_fields.end());
     return arrow::schema(prepared_fields);
 }
@@ -105,14 +103,6 @@ class MergedKeyValueRecordReaderTest : public testing::Test {
     std::shared_ptr<ReducerMergeFunctionWrapper> merge_function_wrapper_;
 };
 
-TEST_F(MergedKeyValueRecordReaderTest, TestRealtimeOffsetField) {
-    const DataField& field = RealtimeOffsetField();
-    ASSERT_EQ(SpecialFieldIds::REALTIME_OFFSET, field.Id());
-    ASSERT_EQ("_REALTIME_OFFSET", field.Name());
-    ASSERT_EQ(arrow::Type::INT64, field.Type()->id());
-    ASSERT_FALSE(field.Nullable());
-}
-
 TEST_F(MergedKeyValueRecordReaderTest, TestMergeAcrossUnderlyingBatches) {
     std::vector<DataField> fields = {DataField(0, arrow::field("k0", arrow::int32())),
                                      DataField(1, arrow::field("k1", arrow::int32())),
diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp
index ee6445057..737ece080 100644
--- a/src/paimon/core/operation/key_value_file_store_write.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write.cpp
@@ -36,7 +36,6 @@
 #include "paimon/core/operation/file_store_scan.h"
 #include "paimon/core/operation/key_value_file_store_scan.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
-#include "paimon/core/realtime/realtime_fields.h"
 #include "paimon/core/realtime/realtime_primary_key_writer.h"
 #include "paimon/core/schema/table_schema.h"
 #include "paimon/core/utils/file_store_path_factory.h"
@@ -136,16 +135,16 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
         partition_map =
             std::map<std::string, std::string>(partition_values.begin(), partition_values.end());
         PAIMON_ASSIGN_OR_RAISE(realtime_context_impl, RealtimeContextImpl::Cast(realtime_context_));
-        if (schema_->GetFieldByName(RealtimeOffsetField().Name())) {
+        if (schema_->GetFieldByName(SpecialFields::RealtimeOffset().Name())) {
             return Status::Invalid("PK real-time write schema contains reserved transport field " +
-                                   RealtimeOffsetField().Name());
+                                   SpecialFields::RealtimeOffset().Name());
         }
         arrow::FieldVector prepared_fields = {
             DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())
                 ->WithNullable(false),
             DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
                 ->WithNullable(false),
-            DataField::ConvertDataFieldToArrowField(RealtimeOffsetField())->WithNullable(false)};
+            DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset())};
         prepared_fields.insert(prepared_fields.end(), schema_->fields().begin(),
                                schema_->fields().end());
         auto c_write_schema = std::make_unique<ArrowSchema>();
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index 864456818..623e3f3fd 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -47,7 +47,6 @@
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/fields_comparator.h"
 #include "paimon/common/utils/scope_guard.h"
-#include "paimon/core/realtime/realtime_fields.h"
 #include "paimon/core/utils/nested_projection_utils.h"
 #include "paimon/macros.h"
 #include "paimon/reader/batch_reader.h"
@@ -672,7 +671,7 @@ Status ValidatePreparedTransportSchema(const std::shared_ptr<arrow::Schema>& pre
     PAIMON_RETURN_NOT_OK(
         CheckPreparedField(prepared_schema, kSequenceNumberIndex, SpecialFields::SequenceNumber()));
     PAIMON_RETURN_NOT_OK(
-        CheckPreparedField(prepared_schema, kRealtimeOffsetIndex, RealtimeOffsetField()));
+        CheckPreparedField(prepared_schema, kRealtimeOffsetIndex, SpecialFields::RealtimeOffset()));
     return Status::OK();
 }
 
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index dc2ce86b1..384b937cb 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -34,7 +34,6 @@
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/core/core_options.h"
-#include "paimon/core/realtime/realtime_fields.h"
 #include "paimon/core/schema/table_schema.h"
 #include "paimon/macros.h"
 #include "paimon/memory/memory_pool.h"
@@ -48,7 +47,7 @@ std::shared_ptr<arrow::Schema> PreparedSchema() {
         {DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
          DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
              ->WithNullable(false),
-         DataField::ConvertDataFieldToArrowField(RealtimeOffsetField())->WithNullable(false),
+         DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()),
          DataField::ConvertDataFieldToArrowField(DataField(0, arrow::field("id", arrow::int64()))),
          DataField::ConvertDataFieldToArrowField(
              DataField(1, arrow::field("value", arrow::utf8())))});
@@ -59,7 +58,7 @@ std::shared_ptr<arrow::Schema> NestedPreparedSchema() {
         {DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
          DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
              ->WithNullable(false),
-         DataField::ConvertDataFieldToArrowField(RealtimeOffsetField())->WithNullable(false),
+         DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()),
          DataField::ConvertDataFieldToArrowField(DataField(0, arrow::field("id", arrow::int64()))),
          DataField::ConvertDataFieldToArrowField(DataField(
              1,
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index 736ebb02d..9b12aa5bd 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -161,8 +161,7 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     RealtimeStoreCreateConfig mode_config = request.mode_config;
     Result<std::shared_ptr<RealtimeStore>> store_result = factory_->Create(std::move(request));
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> store, std::move(store_result));
-    stores_.emplace(key,
-                    RealtimeStoreRegistryEntry{store, requested_schema, std::move(mode_config)});
+    stores_.emplace(key, StoreEntry{store, requested_schema, std::move(mode_config)});
     if (offset_iter != committed_offsets_.end()) {
         reclaimed_offsets_.emplace(key, offset_iter->second);
     }
@@ -172,12 +171,11 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
 int64_t RealtimeContextImpl::AdvanceMaterializedMaxSequenceNumber(
     const RealtimePartitionBucket& partition_bucket, int64_t max_sequence_number) {
     std::lock_guard<std::mutex> lock(mutex_);
-    auto [iter, inserted] =
-        materialized_max_sequence_numbers_.emplace(partition_bucket, max_sequence_number);
-    if (!inserted && max_sequence_number > iter->second) {
-        iter->second = max_sequence_number;
+    StoreEntry& entry = stores_.at(partition_bucket);
+    if (max_sequence_number > entry.materialized_max_sequence_number) {
+        entry.materialized_max_sequence_number = max_sequence_number;
     }
-    return iter->second;
+    return entry.materialized_max_sequence_number;
 }
 
 Result<std::vector<RealtimePartitionBucketView>> RealtimeContextImpl::AcquireReadViews() {
diff --git a/src/paimon/core/realtime/realtime_context_impl.h b/src/paimon/core/realtime/realtime_context_impl.h
index f5118c18f..f0014176d 100644
--- a/src/paimon/core/realtime/realtime_context_impl.h
+++ b/src/paimon/core/realtime/realtime_context_impl.h
@@ -59,12 +59,6 @@ struct RealtimePartitionBucketView {
     std::shared_ptr<RealtimeReadView> read_view;
 };
 
-struct RealtimeStoreRegistryEntry {
-    std::shared_ptr<RealtimeStore> store;
-    std::shared_ptr<arrow::Schema> write_schema;
-    RealtimeStoreCreateConfig mode_config;
-};
-
 class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
  public:
     static Result<std::shared_ptr<RealtimeContextImpl>> Create(
@@ -102,6 +96,13 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
         std::chrono::steady_clock::time_point expire_at;
     };
 
+    struct StoreEntry {
+        std::shared_ptr<RealtimeStore> store;
+        std::shared_ptr<arrow::Schema> write_schema;
+        RealtimeStoreCreateConfig mode_config;
+        int64_t materialized_max_sequence_number = -1;
+    };
+
     explicit RealtimeContextImpl(const std::shared_ptr<RealtimeStoreFactory>& factory);
 
     Status Start();
@@ -111,8 +112,7 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
     std::shared_ptr<RealtimeStoreFactory> factory_;
     std::mutex mutex_;
     std::mutex progress_mutex_;
-    std::map<RealtimePartitionBucket, RealtimeStoreRegistryEntry> stores_;
-    std::map<RealtimePartitionBucket, int64_t> materialized_max_sequence_numbers_;
+    std::map<RealtimePartitionBucket, StoreEntry> stores_;
     // Full-table progress used as the initial offset when a store is created lazily.
     RealtimeOffsetMap committed_offsets_;
     // Progress already reflected in stores owned by this context.
diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp
index 916b46aad..5dc5d8b4f 100644
--- a/src/paimon/core/realtime/realtime_context_test.cpp
+++ b/src/paimon/core/realtime/realtime_context_test.cpp
@@ -189,6 +189,9 @@ TEST(RealtimeContextTest, TestReconcilesPrimaryKeyInitialSequence) {
     const std::map<std::string, std::string> partition = {{"dt", "2026-08-02"}};
     const RealtimePartitionBucket partition_bucket(partition, /*bucket=*/0);
 
+    ASSERT_OK(
+        GetOrCreateAppendStore(context, partition, 0, MakeWriteSchema(), {}, GetDefaultPool()));
+
     ASSERT_EQ(4, context->AdvanceMaterializedMaxSequenceNumber(partition_bucket,
                                                                /*max_sequence_number=*/4));
     ASSERT_EQ(8, context->AdvanceMaterializedMaxSequenceNumber(partition_bucket,
@@ -243,6 +246,28 @@ TEST(RealtimeContextTest, TestCommittedProgressIsMonotonicAndSelective) {
     ASSERT_EQ(std::vector<int64_t>({8}), factory->stores[1]->committed_offsets);
 }
 
+TEST(RealtimeContextTest, TestRemovedInactivePartitionDoesNotRequireReopen) {
+    auto factory = std::make_shared<TestingRealtimeStoreFactory>();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
+    const std::map<std::string, std::string> active_partition = {{"dt", "2026-08-02"}};
+    const std::map<std::string, std::string> inactive_partition = {{"dt", "2026-08-03"}};
+    const RealtimePartitionBucket active_partition_bucket(active_partition, /*bucket=*/0);
+    const RealtimePartitionBucket inactive_partition_bucket(inactive_partition, /*bucket=*/0);
+
+    ASSERT_OK(context->AdvanceCommittedProgress(
+        5, {{active_partition_bucket, /*offset=*/7}, {inactive_partition_bucket, /*offset=*/9}}));
+    ASSERT_OK_AND_ASSIGN(RealtimeStoreState active_state,
+                         GetOrCreateAppendStore(context, active_partition, 0, MakeWriteSchema(), {},
+                                                GetDefaultPool()));
+    ASSERT_EQ(7, active_state.initial_offset);
+
+    ASSERT_OK(context->AdvanceCommittedProgress(6, {{active_partition_bucket, /*offset=*/7}}));
+    ASSERT_OK_AND_ASSIGN(RealtimeStoreState inactive_state,
+                         GetOrCreateAppendStore(context, inactive_partition, 0, MakeWriteSchema(),
+                                                {}, GetDefaultPool()));
+    ASSERT_EQ(0, inactive_state.initial_offset);
+}
+
 TEST(RealtimeContextTest, TestRetriesOnlyIncompleteReclamation) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
@@ -279,6 +304,39 @@ TEST(RealtimeContextTest, TestRetriesOnlyIncompleteReclamation) {
     ASSERT_EQ(std::vector<int64_t>({8}), factory->stores[1]->committed_offsets);
 }
 
+TEST(RealtimeContextTest, TestRequiresReopenWhenCommittedProgressMovesBackwards) {
+    auto factory = std::make_shared<TestingRealtimeStoreFactory>();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
+    const std::map<std::string, std::string> first_partition = {{"dt", "2026-08-02"}};
+    const std::map<std::string, std::string> second_partition = {{"dt", "2026-08-03"}};
+    const RealtimePartitionBucket first_partition_bucket(first_partition, /*bucket=*/0);
+    const RealtimePartitionBucket second_partition_bucket(second_partition, /*bucket=*/0);
+
+    ASSERT_OK(GetOrCreateAppendStore(context, first_partition, 0, MakeWriteSchema(), {},
+                                     GetDefaultPool()));
+    ASSERT_OK(GetOrCreateAppendStore(context, second_partition, 0, MakeWriteSchema(), {},
+                                     GetDefaultPool()));
+    ASSERT_OK(context->AdvanceCommittedProgress(
+        5, {{first_partition_bucket, /*offset=*/7}, {second_partition_bucket, /*offset=*/9}}));
+    ASSERT_EQ(std::vector<int64_t>({7}), factory->stores[0]->committed_offsets);
+    ASSERT_EQ(std::vector<int64_t>({9}), factory->stores[1]->committed_offsets);
+
+    ASSERT_NOK_WITH_MSG(
+        context->AdvanceCommittedProgress(
+            6, {{first_partition_bucket, /*offset=*/6}, {second_partition_bucket, /*offset=*/10}}),
+        "recreate RealtimeContext");
+    ASSERT_NOK_WITH_MSG(
+        context->AdvanceCommittedProgress(6, {{first_partition_bucket, /*offset=*/10}}),
+        "recreate RealtimeContext");
+    ASSERT_EQ(std::vector<int64_t>({7}), factory->stores[0]->committed_offsets);
+    ASSERT_EQ(std::vector<int64_t>({9}), factory->stores[1]->committed_offsets);
+
+    ASSERT_OK(context->AdvanceCommittedProgress(
+        6, {{first_partition_bucket, /*offset=*/10}, {second_partition_bucket, /*offset=*/11}}));
+    ASSERT_EQ(std::vector<int64_t>({7, 10}), factory->stores[0]->committed_offsets);
+    ASSERT_EQ(std::vector<int64_t>({9, 11}), factory->stores[1]->committed_offsets);
+}
+
 TEST(RealtimeContextTest, TestPinsResolvesAndReleasesReadViewTicket) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
diff --git a/src/paimon/core/realtime/realtime_fields.h b/src/paimon/core/realtime/realtime_fields.h
deleted file mode 100644
index 270941238..000000000
--- a/src/paimon/core/realtime/realtime_fields.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include "arrow/type.h"
-#include "paimon/common/types/data_field.h"
-#include "paimon/utils/special_field_ids.h"
-
-namespace paimon {
-
-inline const DataField& RealtimeOffsetField() {
-    static const DataField data_field =
-        DataField(SpecialFieldIds::REALTIME_OFFSET,
-                  arrow::field("_REALTIME_OFFSET", arrow::int64(), /*nullable=*/false));
-    return data_field;
-}
-
-}  // namespace paimon
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index 185156eb7..7f4d4b5f7 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -38,7 +38,6 @@
 #include "paimon/core/mergetree/merge_tree_writer.h"
 #include "paimon/core/realtime/prepared_key_value_reader.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
-#include "paimon/core/realtime/realtime_fields.h"
 #include "paimon/core/utils/commit_increment.h"
 #include "paimon/macros.h"
 
@@ -169,7 +168,7 @@ Result<std::shared_ptr<RealtimePrimaryKeyWriter>> RealtimePrimaryKeyWriter::Crea
         DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
         DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
             ->WithNullable(false),
-        DataField::ConvertDataFieldToArrowField(RealtimeOffsetField())->WithNullable(false)};
+        DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset())};
     prepared_fields.insert(prepared_fields.end(), write_schema->fields().begin(),
                            write_schema->fields().end());
     const RealtimePartitionBucket partition_bucket(partition, bucket);
diff --git a/src/paimon/core/table/source/append_only_table_read.cpp b/src/paimon/core/table/source/append_only_table_read.cpp
index 34c6ef850..12b3823a7 100644
--- a/src/paimon/core/table/source/append_only_table_read.cpp
+++ b/src/paimon/core/table/source/append_only_table_read.cpp
@@ -111,10 +111,7 @@ Result<std::unique_ptr<BatchReader>> AppendOnlyTableRead::CreateReader(
                 realtime_context_impl->ReleaseReadView(realtime_split->OpaqueTicket()));
         }
     }
-    std::unique_ptr<BatchReader> result =
-        std::make_unique<ConcatBatchReader>(std::move(readers), GetMemoryPool());
-    cleanup_guard.Release();
-    return result;
+    return std::make_unique<ConcatBatchReader>(std::move(readers), GetMemoryPool());
 }
 
 Result<std::unique_ptr<BatchReader>> AppendOnlyTableRead::CreateRealtimeReader(
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index af585a1bb..32231140e 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -38,7 +38,6 @@
 #include "paimon/core/operation/raw_file_split_read.h"
 #include "paimon/core/realtime/prepared_key_value_reader.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
-#include "paimon/core/realtime/realtime_fields.h"
 #include "paimon/core/realtime/realtime_reader.h"
 #include "paimon/core/table/source/data_split_impl.h"
 #include "paimon/core/table/source/pk_count_reader.h"
@@ -68,7 +67,7 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateMemoryReaders(
         DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
         DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
             ->WithNullable(false),
-        DataField::ConvertDataFieldToArrowField(RealtimeOffsetField())->WithNullable(false)};
+        DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset())};
     prepared_fields.insert(prepared_fields.end(), full_value_schema->fields().begin(),
                            full_value_schema->fields().end());
     std::shared_ptr<arrow::Schema> prepared_schema = arrow::schema(std::move(prepared_fields));
@@ -243,10 +242,7 @@ Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateReader(
                 realtime_context_impl->ReleaseReadView(realtime_split->OpaqueTicket()));
         }
     }
-    std::unique_ptr<BatchReader> result =
-        std::make_unique<ConcatBatchReader>(std::move(readers), GetMemoryPool());
-    cleanup_guard.Release();
-    return result;
+    return std::make_unique<ConcatBatchReader>(std::move(readers), GetMemoryPool());
 }
 
 Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateRealtimeReader(
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index 918d1dd35..7c143c995 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -23,6 +23,7 @@
 #include <condition_variable>
 #include <cstdint>
 #include <deque>
+#include <functional>
 #include <iterator>
 #include <limits>
 #include <map>
@@ -177,12 +178,10 @@ class ReadViewCheckingBatchReader final : public BatchReader {
     std::weak_ptr<RealtimeReadView> read_view_;
 };
 
-class QueryTrackingRealtimeStore final : public RealtimeStore {
+class DelegatingRealtimeStore : public RealtimeStore {
  public:
-    QueryTrackingRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
-                               const std::shared_ptr<std::atomic<bool>>& saw_query_predicate,
-                               const std::shared_ptr<std::weak_ptr<RealtimeReadView>>& query_view)
-        : delegate_(delegate), saw_query_predicate_(saw_query_predicate), query_view_(query_view) {}
+    explicit DelegatingRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate)
+        : delegate_(delegate) {}
 
     Status Write(RealtimeWriteBatch&& batch) override {
         return delegate_->Write(std::move(batch));
@@ -197,6 +196,64 @@ class QueryTrackingRealtimeStore final : public RealtimeStore {
         return delegate_->CreateCommitReaders(segment);
     }
 
+    Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
+        return delegate_->AcquireReadView();
+    }
+
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
+        const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
+        const RealtimeQueryContext& context) override {
+        return delegate_->CreateQueryReaders(view, offset_begin, context);
+    }
+
+    Status AdvanceCommittedOffset(int64_t committed_offset) override {
+        return delegate_->AdvanceCommittedOffset(committed_offset);
+    }
+
+    uint64_t GetMemoryUsage() const override {
+        return delegate_->GetMemoryUsage();
+    }
+
+ protected:
+    std::shared_ptr<RealtimeStore> delegate_;
+};
+
+class DecoratingRealtimeStoreFactory final : public RealtimeStoreFactory {
+ public:
+    using Decorator =
+        std::function<std::shared_ptr<RealtimeStore>(const std::shared_ptr<RealtimeStore>&)>;
+
+    explicit DecoratingRealtimeStoreFactory(Decorator decorator)
+        : decorator_(std::move(decorator)) {}
+
+    Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> delegate,
+                               delegate_.Create(std::move(request)));
+        return decorator_(delegate);
+    }
+
+ private:
+    ArrowRealtimeStoreFactory delegate_;
+    Decorator decorator_;
+};
+
+template <typename Store, typename... Args>
+std::shared_ptr<RealtimeStoreFactory> MakeDecoratingFactory(Args... args) {
+    return std::make_shared<DecoratingRealtimeStoreFactory>(
+        [=](const std::shared_ptr<RealtimeStore>& delegate) -> std::shared_ptr<RealtimeStore> {
+            return std::make_shared<Store>(delegate, args...);
+        });
+}
+
+class QueryTrackingRealtimeStore final : public DelegatingRealtimeStore {
+ public:
+    QueryTrackingRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
+                               const std::shared_ptr<std::atomic<bool>>& saw_query_predicate,
+                               const std::shared_ptr<std::weak_ptr<RealtimeReadView>>& query_view)
+        : DelegatingRealtimeStore(delegate),
+          saw_query_predicate_(saw_query_predicate),
+          query_view_(query_view) {}
+
     Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeReadView> delegate_view,
                                delegate_->AcquireReadView());
@@ -225,36 +282,7 @@ class QueryTrackingRealtimeStore final : public RealtimeStore {
         return readers;
     }
 
-    Status AdvanceCommittedOffset(int64_t committed_offset) override {
-        return delegate_->AdvanceCommittedOffset(committed_offset);
-    }
-
-    uint64_t GetMemoryUsage() const override {
-        return delegate_->GetMemoryUsage();
-    }
-
  private:
-    std::shared_ptr<RealtimeStore> delegate_;
-    std::shared_ptr<std::atomic<bool>> saw_query_predicate_;
-    std::shared_ptr<std::weak_ptr<RealtimeReadView>> query_view_;
-};
-
-class QueryTrackingRealtimeStoreFactory final : public RealtimeStoreFactory {
- public:
-    explicit QueryTrackingRealtimeStoreFactory(
-        const std::shared_ptr<std::atomic<bool>>& saw_query_predicate,
-        const std::shared_ptr<std::weak_ptr<RealtimeReadView>>& query_view)
-        : saw_query_predicate_(saw_query_predicate), query_view_(query_view) {}
-
-    Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> delegate,
-                               delegate_.Create(std::move(request)));
-        return std::shared_ptr<RealtimeStore>(std::make_shared<QueryTrackingRealtimeStore>(
-            delegate, saw_query_predicate_, query_view_));
-    }
-
- private:
-    ArrowRealtimeStoreFactory delegate_;
     std::shared_ptr<std::atomic<bool>> saw_query_predicate_;
     std::shared_ptr<std::weak_ptr<RealtimeReadView>> query_view_;
 };
@@ -292,19 +320,11 @@ struct CloseTrackingReaderState {
     int32_t commit_null_index = -1;
 };
 
-class CloseTrackingRealtimeStore final : public RealtimeStore {
+class CloseTrackingRealtimeStore final : public DelegatingRealtimeStore {
  public:
     CloseTrackingRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
                                const std::shared_ptr<CloseTrackingReaderState>& state)
-        : delegate_(delegate), state_(state) {}
-
-    Status Write(RealtimeWriteBatch&& batch) override {
-        return delegate_->Write(std::move(batch));
-    }
-
-    Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>> SealForCommit() override {
-        return delegate_->SealForCommit();
-    }
+        : DelegatingRealtimeStore(delegate), state_(state) {}
 
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
         const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
@@ -318,10 +338,6 @@ class CloseTrackingRealtimeStore final : public RealtimeStore {
         return readers;
     }
 
-    Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
-        return delegate_->AcquireReadView();
-    }
-
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
         const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
         const RealtimeQueryContext& context) override {
@@ -335,14 +351,6 @@ class CloseTrackingRealtimeStore final : public RealtimeStore {
         return readers;
     }
 
-    Status AdvanceCommittedOffset(int64_t committed_offset) override {
-        return delegate_->AdvanceCommittedOffset(committed_offset);
-    }
-
-    uint64_t GetMemoryUsage() const override {
-        return delegate_->GetMemoryUsage();
-    }
-
  private:
     static Status InsertNullReader(int32_t index,
                                    std::vector<std::unique_ptr<BatchReader>>* readers) {
@@ -356,25 +364,6 @@ class CloseTrackingRealtimeStore final : public RealtimeStore {
         return Status::OK();
     }
 
-    std::shared_ptr<RealtimeStore> delegate_;
-    std::shared_ptr<CloseTrackingReaderState> state_;
-};
-
-class CloseTrackingRealtimeStoreFactory final : public RealtimeStoreFactory {
- public:
-    explicit CloseTrackingRealtimeStoreFactory(
-        const std::shared_ptr<CloseTrackingReaderState>& state)
-        : state_(state) {}
-
-    Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> delegate,
-                               delegate_.Create(std::move(request)));
-        return std::shared_ptr<RealtimeStore>(
-            std::make_shared<CloseTrackingRealtimeStore>(delegate, state_));
-    }
-
- private:
-    ArrowRealtimeStoreFactory delegate_;
     std::shared_ptr<CloseTrackingReaderState> state_;
 };
 
@@ -421,18 +410,10 @@ class SplitBatchReader final : public BatchReader {
     int64_t next_row_ = 0;
 };
 
-class SplitCommitReaderRealtimeStore final : public RealtimeStore {
+class SplitCommitReaderRealtimeStore final : public DelegatingRealtimeStore {
  public:
     explicit SplitCommitReaderRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate)
-        : delegate_(delegate) {}
-
-    Status Write(RealtimeWriteBatch&& batch) override {
-        return delegate_->Write(std::move(batch));
-    }
-
-    Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>> SealForCommit() override {
-        return delegate_->SealForCommit();
-    }
+        : DelegatingRealtimeStore(delegate) {}
 
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
         const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
@@ -443,48 +424,39 @@ class SplitCommitReaderRealtimeStore final : public RealtimeStore {
         }
         return readers;
     }
+};
 
-    Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
-        return delegate_->AcquireReadView();
-    }
+enum class CommitReaderMalformation { DROP_LAST, UNSORTED, SUBSTITUTE_OFFSET };
 
-    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
-        const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
-        const RealtimeQueryContext& context) override {
-        return delegate_->CreateQueryReaders(view, offset_begin, context);
-    }
+class CorruptingBatchReader final : public BatchReader {
+ public:
+    CorruptingBatchReader(std::unique_ptr<BatchReader> delegate,
+                          CommitReaderMalformation malformation)
+        : delegate_(std::move(delegate)), malformation_(malformation) {}
 
-    Status AdvanceCommittedOffset(int64_t committed_offset) override {
-        return delegate_->AdvanceCommittedOffset(committed_offset);
+    Result<ReadBatch> NextBatch() override {
+        switch (malformation_) {
+            case CommitReaderMalformation::DROP_LAST:
+                return DropLast();
+            case CommitReaderMalformation::UNSORTED:
+                return SwapFirstTwo();
+            case CommitReaderMalformation::SUBSTITUTE_OFFSET:
+                return SubstituteOffset();
+        }
+        return Status::Invalid("unknown commit reader malformation");
     }
 
-    uint64_t GetMemoryUsage() const override {
-        return delegate_->GetMemoryUsage();
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return delegate_->GetReaderMetrics();
     }
 
- private:
-    std::shared_ptr<RealtimeStore> delegate_;
-};
-
-class SplitCommitReaderRealtimeStoreFactory final : public RealtimeStoreFactory {
- public:
-    Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> delegate,
-                               delegate_.Create(std::move(request)));
-        return std::shared_ptr<RealtimeStore>(
-            std::make_shared<SplitCommitReaderRealtimeStore>(delegate));
+    void Close() override {
+        buffered_.reset();
+        delegate_->Close();
     }
 
  private:
-    ArrowRealtimeStoreFactory delegate_;
-};
-
-class DropLastBatchReader final : public BatchReader {
- public:
-    explicit DropLastBatchReader(std::unique_ptr<BatchReader> delegate)
-        : delegate_(std::move(delegate)) {}
-
-    Result<ReadBatch> NextBatch() override {
+    Result<ReadBatch> DropLast() {
         if (!buffered_.has_value()) {
             PAIMON_ASSIGN_OR_RAISE(ReadBatch first, delegate_->NextBatch());
             if (BatchReader::IsEofBatch(first)) {
@@ -502,72 +474,34 @@ class DropLastBatchReader final : public BatchReader {
         return result;
     }
 
-    std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return delegate_->GetReaderMetrics();
-    }
-
-    void Close() override {
-        buffered_.reset();
-        delegate_->Close();
-    }
-
- private:
-    std::unique_ptr<BatchReader> delegate_;
-    std::optional<ReadBatch> buffered_;
-};
-
-class SwapFirstTwoBatchReader final : public BatchReader {
- public:
-    explicit SwapFirstTwoBatchReader(std::unique_ptr<BatchReader> delegate)
-        : delegate_(std::move(delegate)) {}
-
-    Result<ReadBatch> NextBatch() override {
-        if (!initialized_) {
-            initialized_ = true;
-            PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, delegate_->NextBatch());
-            if (BatchReader::IsEofBatch(batch)) {
-                return MakeEofBatch();
-            }
-            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-                std::shared_ptr<arrow::Array> array,
-                arrow::ImportArray(batch.first.get(), batch.second.get()));
-            if (array->length() < 2) {
-                return Status::Invalid("cannot make a one-row reader unsorted");
-            }
-            arrow::ArrayVector pieces = {array->Slice(1, 1), array->Slice(0, 1)};
-            if (array->length() > 2) {
-                pieces.push_back(array->Slice(2));
-            }
-            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> swapped,
-                                              arrow::Concatenate(pieces));
-            auto output = std::make_unique<ArrowArray>();
-            auto schema = std::make_unique<ArrowSchema>();
-            PAIMON_RETURN_NOT_OK_FROM_ARROW(
-                arrow::ExportArray(*swapped, output.get(), schema.get()));
-            return ReadBatch(std::move(output), std::move(schema));
+    Result<ReadBatch> SwapFirstTwo() {
+        if (corrupted_) {
+            return delegate_->NextBatch();
         }
-        return delegate_->NextBatch();
-    }
-
-    std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return delegate_->GetReaderMetrics();
-    }
-
-    void Close() override {
-        delegate_->Close();
+        corrupted_ = true;
+        PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, delegate_->NextBatch());
+        if (BatchReader::IsEofBatch(batch)) {
+            return MakeEofBatch();
+        }
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+            std::shared_ptr<arrow::Array> array,
+            arrow::ImportArray(batch.first.get(), batch.second.get()));
+        if (array->length() < 2) {
+            return Status::Invalid("cannot make a one-row reader unsorted");
+        }
+        arrow::ArrayVector pieces = {array->Slice(1, 1), array->Slice(0, 1)};
+        if (array->length() > 2) {
+            pieces.push_back(array->Slice(2));
+        }
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> swapped,
+                                          arrow::Concatenate(pieces));
+        auto output = std::make_unique<ArrowArray>();
+        auto schema = std::make_unique<ArrowSchema>();
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*swapped, output.get(), schema.get()));
+        return ReadBatch(std::move(output), std::move(schema));
     }
 
- private:
-    bool initialized_ = false;
-    std::unique_ptr<BatchReader> delegate_;
-};
-
-class SubstituteOffsetBatchReader final : public BatchReader {
- public:
-    explicit SubstituteOffsetBatchReader(std::unique_ptr<BatchReader> delegate)
-        : delegate_(std::move(delegate)) {}
-
-    Result<ReadBatch> NextBatch() override {
+    Result<ReadBatch> SubstituteOffset() {
         PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, delegate_->NextBatch());
         if (BatchReader::IsEofBatch(batch)) {
             return batch;
@@ -602,86 +536,29 @@ class SubstituteOffsetBatchReader final : public BatchReader {
         return ReadBatch(std::move(output), std::move(schema));
     }
 
-    std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return delegate_->GetReaderMetrics();
-    }
-
-    void Close() override {
-        delegate_->Close();
-    }
-
- private:
     std::unique_ptr<BatchReader> delegate_;
+    CommitReaderMalformation malformation_;
+    bool corrupted_ = false;
+    std::optional<ReadBatch> buffered_;
 };
 
-enum class CommitReaderMalformation { DROP_LAST, UNSORTED, SUBSTITUTE_OFFSET };
-
-class MalformedCoverageRealtimeStore final : public RealtimeStore {
+class MalformedCoverageRealtimeStore final : public DelegatingRealtimeStore {
  public:
     MalformedCoverageRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
                                    CommitReaderMalformation malformation)
-        : delegate_(delegate), malformation_(malformation) {}
+        : DelegatingRealtimeStore(delegate), malformation_(malformation) {}
 
-    Status Write(RealtimeWriteBatch&& batch) override {
-        return delegate_->Write(std::move(batch));
-    }
-    Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>> SealForCommit() override {
-        return delegate_->SealForCommit();
-    }
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
         const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
         PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
                                delegate_->CreateCommitReaders(segment));
         for (std::unique_ptr<BatchReader>& reader : readers) {
-            switch (malformation_) {
-                case CommitReaderMalformation::DROP_LAST:
-                    reader = std::make_unique<DropLastBatchReader>(std::move(reader));
-                    break;
-                case CommitReaderMalformation::UNSORTED:
-                    reader = std::make_unique<SwapFirstTwoBatchReader>(std::move(reader));
-                    break;
-                case CommitReaderMalformation::SUBSTITUTE_OFFSET:
-                    reader = std::make_unique<SubstituteOffsetBatchReader>(std::move(reader));
-                    break;
-            }
+            reader = std::make_unique<CorruptingBatchReader>(std::move(reader), malformation_);
         }
         return readers;
     }
-    Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
-        return delegate_->AcquireReadView();
-    }
-    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
-        const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
-        const RealtimeQueryContext& context) override {
-        return delegate_->CreateQueryReaders(view, offset_begin, context);
-    }
-    Status AdvanceCommittedOffset(int64_t committed_offset) override {
-        return delegate_->AdvanceCommittedOffset(committed_offset);
-    }
-    uint64_t GetMemoryUsage() const override {
-        return delegate_->GetMemoryUsage();
-    }
 
  private:
-    std::shared_ptr<RealtimeStore> delegate_;
-    CommitReaderMalformation malformation_;
-};
-
-class MalformedCoverageRealtimeStoreFactory final : public RealtimeStoreFactory {
- public:
-    explicit MalformedCoverageRealtimeStoreFactory(
-        CommitReaderMalformation malformation = CommitReaderMalformation::DROP_LAST)
-        : malformation_(malformation) {}
-
-    Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> delegate,
-                               delegate_.Create(std::move(request)));
-        return std::shared_ptr<RealtimeStore>(
-            std::make_shared<MalformedCoverageRealtimeStore>(delegate, malformation_));
-    }
-
- private:
-    ArrowRealtimeStoreFactory delegate_;
     CommitReaderMalformation malformation_;
 };
 
@@ -1062,16 +939,6 @@ class RealtimeWriteInteTest : public ::testing::Test {
         return commit->Expire();
     }
 
-    Status CommitMessages(const std::vector<std::shared_ptr<CommitMessage>>& messages,
-                          int64_t commit_identifier) const {
-        CommitContextBuilder builder(table_path_, commit_user_);
-        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<CommitContext> context,
-                               builder.SetOptions(options_).Finish());
-        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreCommit> commit,
-                               FileStoreCommit::Create(std::move(context)));
-        return commit->Commit(messages, commit_identifier);
-    }
-
     Result<std::shared_ptr<Plan>> CreatePlan(
         const std::shared_ptr<RealtimeContext>& realtime_context,
         const std::shared_ptr<Predicate>& predicate) const {
@@ -1413,7 +1280,7 @@ class RealtimeWriteInteTest : public ::testing::Test {
             CreateTable(/*partition_keys=*/{"pt"});
         }
         auto close_state = std::make_shared<CloseTrackingReaderState>();
-        auto factory = std::make_shared<CloseTrackingRealtimeStoreFactory>(close_state);
+        auto factory = MakeDecoratingFactory<CloseTrackingRealtimeStore>(close_state);
         ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                              RealtimeContext::Create(factory));
         ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
@@ -1524,7 +1391,7 @@ TEST_F(RealtimeWriteInteTest, TestPkRead) {
     auto saw_query_predicate = std::make_shared<std::atomic<bool>>(false);
     auto query_view = std::make_shared<std::weak_ptr<RealtimeReadView>>();
     auto factory =
-        std::make_shared<QueryTrackingRealtimeStoreFactory>(saw_query_predicate, query_view);
+        MakeDecoratingFactory<QueryTrackingRealtimeStore>(saw_query_predicate, query_view);
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                          RealtimeContext::Create(factory));
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
@@ -2257,7 +2124,12 @@ TEST_F(RealtimeWriteInteTest, TestPkCompaction) {
             std::max(compacted_live_max_sequence_number, file->max_sequence_number);
     }
     ASSERT_LT(compacted_live_max_sequence_number, kHistoricalMaxSequenceNumber);
-    ASSERT_OK(CommitMessages(compact_messages, /*commit_identifier=*/4));
+    CommitContextBuilder commit_builder(table_path_, commit_user_);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
+                         commit_builder.SetOptions(options_).Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> commit,
+                         FileStoreCommit::Create(std::move(commit_context)));
+    ASSERT_OK(commit->Commit(compact_messages, /*commit_identifier=*/4));
     ASSERT_OK(compact_writer->Close());
 
     ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(options_));
@@ -2304,7 +2176,7 @@ TEST_F(RealtimeWriteInteTest, TestPkCompaction) {
 
 TEST_F(RealtimeWriteInteTest, TestPkPluginContract) {
     CreatePkTable();
-    auto factory = std::make_shared<SplitCommitReaderRealtimeStoreFactory>();
+    auto factory = MakeDecoratingFactory<SplitCommitReaderRealtimeStore>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                          RealtimeContext::Create(factory));
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
@@ -2332,7 +2204,8 @@ TEST_F(RealtimeWriteInteTest, TestPkPluginContract) {
 
 TEST_F(RealtimeWriteInteTest, TestPkRejectsMalformedCoverage) {
     CreatePkTable();
-    auto factory = std::make_shared<MalformedCoverageRealtimeStoreFactory>();
+    auto factory =
+        MakeDecoratingFactory<MalformedCoverageRealtimeStore>(CommitReaderMalformation::DROP_LAST);
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                          RealtimeContext::Create(factory));
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
@@ -2348,7 +2221,7 @@ TEST_F(RealtimeWriteInteTest, TestPkRejectsMalformedCoverage) {
 
 TEST_F(RealtimeWriteInteTest, TestPkRejectsEqualCardinalityOffsetSubstitution) {
     CreatePkTable();
-    auto factory = std::make_shared<MalformedCoverageRealtimeStoreFactory>(
+    auto factory = MakeDecoratingFactory<MalformedCoverageRealtimeStore>(
         CommitReaderMalformation::SUBSTITUTE_OFFSET);
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                          RealtimeContext::Create(factory));
@@ -2366,7 +2239,7 @@ TEST_F(RealtimeWriteInteTest, TestPkRejectsEqualCardinalityOffsetSubstitution) {
 TEST_F(RealtimeWriteInteTest, TestPkRejectsUnsortedPluginRows) {
     CreatePkTable();
     auto factory =
-        std::make_shared<MalformedCoverageRealtimeStoreFactory>(CommitReaderMalformation::UNSORTED);
+        MakeDecoratingFactory<MalformedCoverageRealtimeStore>(CommitReaderMalformation::UNSORTED);
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                          RealtimeContext::Create(factory));
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
@@ -2383,7 +2256,7 @@ TEST_F(RealtimeWriteInteTest, TestPkRejectsUnsortedPluginRows) {
 TEST_F(RealtimeWriteInteTest, TestPkQueryReaderClose) {
     CreatePkTable();
     auto state = std::make_shared<CloseTrackingReaderState>();
-    auto factory = std::make_shared<CloseTrackingRealtimeStoreFactory>(state);
+    auto factory = MakeDecoratingFactory<CloseTrackingRealtimeStore>(state);
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                          RealtimeContext::Create(factory));
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
@@ -2421,7 +2294,7 @@ TEST_F(RealtimeWriteInteTest, TestPkQueryReaderClose) {
 TEST_F(RealtimeWriteInteTest, TestPkQueryReaderCloseFailure) {
     CreatePkTable();
     auto state = std::make_shared<CloseTrackingReaderState>();
-    auto factory = std::make_shared<CloseTrackingRealtimeStoreFactory>(state);
+    auto factory = MakeDecoratingFactory<CloseTrackingRealtimeStore>(state);
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                          RealtimeContext::Create(factory));
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
@@ -2462,7 +2335,7 @@ TEST_F(RealtimeWriteInteTest, TestAppendQueryReaderCloseFailure) {
     CreateTable(/*partition_keys=*/{});
     auto state = std::make_shared<CloseTrackingReaderState>();
     state->query_null_index = 1;
-    auto factory = std::make_shared<CloseTrackingRealtimeStoreFactory>(state);
+    auto factory = MakeDecoratingFactory<CloseTrackingRealtimeStore>(state);
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                          RealtimeContext::Create(factory));
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
@@ -2496,7 +2369,7 @@ TEST_F(RealtimeWriteInteTest, TestPkCommitReaderCloseFailure) {
     CreatePkTable();
     auto state = std::make_shared<CloseTrackingReaderState>();
     state->commit_null_index = 1;
-    auto factory = std::make_shared<CloseTrackingRealtimeStoreFactory>(state);
+    auto factory = MakeDecoratingFactory<CloseTrackingRealtimeStore>(state);
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                          RealtimeContext::Create(factory));
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,

From e68ac16f7fb8ccac9c4e09ad1588dab934d4d681 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Tue, 25 Aug 2026 11:43:44 +0800
Subject: [PATCH 27/62] test(realtime): simplify integration test setup

---
 test/inte/realtime_write_inte_test.cpp | 140 ++++++++++---------------
 1 file changed, 57 insertions(+), 83 deletions(-)

diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index 7c143c995..86f8a4a54 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -954,6 +954,27 @@ class RealtimeWriteInteTest : public ::testing::Test {
         return scan->CreatePlan();
     }
 
+    Result<std::unique_ptr<BatchReader>> CreateQueryReader(
+        const std::shared_ptr<Plan>& plan,
+        const std::shared_ptr<RealtimeContext>& realtime_context) const {
+        ReadContextBuilder read_builder(table_path_);
+        read_builder.SetOptions(options_)
+            .SetReadFieldNames({"id", "payload", "pt"})
+            .WithRealtimeContext(realtime_context)
+            .WithMemoryPool(pool_);
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableRead> table_read,
+                               TableRead::Create(std::move(read_context)));
+        return table_read->CreateReader(plan->Splits());
+    }
+
+    Result<std::unique_ptr<BatchReader>> CreateQueryReader(
+        const std::shared_ptr<RealtimeContext>& realtime_context) const {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Plan> plan,
+                               CreatePlan(realtime_context, /*predicate=*/nullptr));
+        return CreateQueryReader(plan, realtime_context);
+    }
+
     Result<CollectedReadResult> ReadPlan(const std::shared_ptr<Plan>& plan,
                                          const std::shared_ptr<RealtimeContext>& realtime_context,
                                          const std::vector<std::string>& read_fields,
@@ -1329,6 +1350,23 @@ class RealtimeWriteInteTest : public ::testing::Test {
         ASSERT_OK(writer->Close());
     }
 
+    void CheckPkRejectsCommitReaderMalformation(CommitReaderMalformation malformation,
+                                                const std::string& expected_error) {
+        CreatePkTable();
+        auto factory = MakeDecoratingFactory<MalformedCoverageRealtimeStore>(malformation);
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                             RealtimeContext::Create(factory));
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                             CreateRealtimeWriter(realtime_context));
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                             MakeBatch({Row{1, "one", "p0"}, Row{2, "two", "p0"}},
+                                       /*partitioned=*/false));
+        ASSERT_OK(writer->Write(std::move(batch)));
+        ASSERT_NOK_WITH_MSG(writer->PrepareCommitWithProgress(/*commit_identifier=*/0),
+                            expected_error);
+        ASSERT_OK(writer->Close());
+    }
+
     std::unique_ptr<UniqueTestDirectory> dir_;
     std::string table_path_;
     std::string commit_user_ = "realtime_commit_user";
@@ -2203,54 +2241,19 @@ TEST_F(RealtimeWriteInteTest, TestPkPluginContract) {
 }
 
 TEST_F(RealtimeWriteInteTest, TestPkRejectsMalformedCoverage) {
-    CreatePkTable();
-    auto factory =
-        MakeDecoratingFactory<MalformedCoverageRealtimeStore>(CommitReaderMalformation::DROP_LAST);
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create(factory));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                         CreateRealtimeWriter(realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
-                         MakeBatch({Row{1, "one", "p0"}, Row{2, "two", "p0"}},
-                                   /*partitioned=*/false));
-    ASSERT_OK(writer->Write(std::move(batch)));
-    ASSERT_NOK_WITH_MSG(writer->PrepareCommitWithProgress(/*commit_identifier=*/0),
-                        "commit readers did not cover the sealed range");
-    ASSERT_OK(writer->Close());
+    CheckPkRejectsCommitReaderMalformation(CommitReaderMalformation::DROP_LAST,
+                                           "commit readers did not cover the sealed range");
 }
 
 TEST_F(RealtimeWriteInteTest, TestPkRejectsEqualCardinalityOffsetSubstitution) {
-    CreatePkTable();
-    auto factory = MakeDecoratingFactory<MalformedCoverageRealtimeStore>(
-        CommitReaderMalformation::SUBSTITUTE_OFFSET);
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create(factory));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                         CreateRealtimeWriter(realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
-                         MakeBatch({Row{1, "one", "p0"}, Row{2, "two", "p0"}},
-                                   /*partitioned=*/false));
-    ASSERT_OK(writer->Write(std::move(batch)));
-    ASSERT_NOK_WITH_MSG(writer->PrepareCommitWithProgress(/*commit_identifier=*/0),
-                        "duplicate REALTIME_OFFSET");
-    ASSERT_OK(writer->Close());
+    CheckPkRejectsCommitReaderMalformation(CommitReaderMalformation::SUBSTITUTE_OFFSET,
+                                           "duplicate REALTIME_OFFSET");
 }
 
 TEST_F(RealtimeWriteInteTest, TestPkRejectsUnsortedPluginRows) {
-    CreatePkTable();
-    auto factory =
-        MakeDecoratingFactory<MalformedCoverageRealtimeStore>(CommitReaderMalformation::UNSORTED);
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create(factory));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                         CreateRealtimeWriter(realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
-                         MakeBatch({Row{1, "one", "p0"}, Row{2, "two", "p0"}},
-                                   /*partitioned=*/false));
-    ASSERT_OK(writer->Write(std::move(batch)));
-    ASSERT_NOK_WITH_MSG(writer->PrepareCommitWithProgress(/*commit_identifier=*/0),
-                        "not globally sorted by primary key and sequence number");
-    ASSERT_OK(writer->Close());
+    CheckPkRejectsCommitReaderMalformation(
+        CommitReaderMalformation::UNSORTED,
+        "not globally sorted by primary key and sequence number");
 }
 
 TEST_F(RealtimeWriteInteTest, TestPkQueryReaderClose) {
@@ -2265,27 +2268,19 @@ TEST_F(RealtimeWriteInteTest, TestPkQueryReaderClose) {
                          MakeBatch({Row{1, "one", "p0"}}, /*partitioned=*/false));
     ASSERT_OK(writer->Write(std::move(batch)));
 
-    auto create_reader = [&]() -> Result<std::unique_ptr<BatchReader>> {
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Plan> plan,
-                               CreatePlan(realtime_context, /*predicate=*/nullptr));
-        ReadContextBuilder read_builder(table_path_);
-        read_builder.SetOptions(options_)
-            .SetReadFieldNames({"id", "payload", "pt"})
-            .WithRealtimeContext(realtime_context)
-            .WithMemoryPool(pool_);
-        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
-        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableRead> table_read,
-                               TableRead::Create(std::move(read_context)));
-        return table_read->CreateReader(plan->Splits());
+    auto release_reader = [&](bool explicit_close) -> Status {
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> reader,
+                               CreateQueryReader(realtime_context));
+        if (explicit_close) {
+            reader->Close();
+        }
+        return Status::OK();
     };
 
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> explicitly_closed_reader, create_reader());
-    explicitly_closed_reader->Close();
-    explicitly_closed_reader.reset();
+    ASSERT_OK(release_reader(/*explicit_close=*/true));
     ASSERT_EQ(1, state->query_close_count->load(std::memory_order_acquire));
 
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> destroyed_reader, create_reader());
-    destroyed_reader.reset();
+    ASSERT_OK(release_reader(/*explicit_close=*/false));
     ASSERT_EQ(2, state->query_close_count->load(std::memory_order_acquire));
 
     ASSERT_OK(writer->Close());
@@ -2309,23 +2304,10 @@ TEST_F(RealtimeWriteInteTest, TestPkQueryReaderCloseFailure) {
                          MakeBatch({Row{2, "two", "p0"}}, /*partitioned=*/false));
     ASSERT_OK(writer->Write(std::move(second_batch)));
 
-    auto create_reader = [&]() -> Result<std::unique_ptr<BatchReader>> {
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Plan> plan,
-                               CreatePlan(realtime_context, /*predicate=*/nullptr));
-        ReadContextBuilder read_builder(table_path_);
-        read_builder.SetOptions(options_)
-            .SetReadFieldNames({"id", "payload", "pt"})
-            .WithRealtimeContext(realtime_context)
-            .WithMemoryPool(pool_);
-        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
-        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableRead> table_read,
-                               TableRead::Create(std::move(read_context)));
-        return table_read->CreateReader(plan->Splits());
-    };
-
     for (int32_t null_index = 0; null_index <= 1; ++null_index) {
         state->query_null_index = null_index;
-        ASSERT_NOK_WITH_MSG(create_reader(), "PK real-time store returned a null query reader");
+        ASSERT_NOK_WITH_MSG(CreateQueryReader(realtime_context),
+                            "PK real-time store returned a null query reader");
         ASSERT_EQ(null_index + 1, state->query_close_count->load(std::memory_order_acquire));
     }
     ASSERT_OK(writer->Close());
@@ -2347,15 +2329,7 @@ TEST_F(RealtimeWriteInteTest, TestAppendQueryReaderCloseFailure) {
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
                          CreatePlan(realtime_context, /*predicate=*/nullptr));
 
-    ReadContextBuilder read_builder(table_path_);
-    read_builder.SetOptions(options_)
-        .SetReadFieldNames({"id", "payload", "pt"})
-        .WithRealtimeContext(realtime_context)
-        .WithMemoryPool(pool_);
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableRead> table_read,
-                         TableRead::Create(std::move(read_context)));
-    ASSERT_NOK_WITH_MSG(table_read->CreateReader(plan->Splits()),
+    ASSERT_NOK_WITH_MSG(CreateQueryReader(plan, realtime_context),
                         "append-only real-time store returned a null query reader");
     ASSERT_EQ(1, state->query_close_count->load(std::memory_order_acquire));
 

From f92fe6a5cafff2092d0102d83a1def57ffd15840 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Tue, 25 Aug 2026 13:18:36 +0800
Subject: [PATCH 28/62] fix(realtime): reject PK read-optimized scans

---
 src/paimon/core/table/source/table_scan.cpp   |   9 +-
 .../system/read_optimized_system_table.cpp    |   4 +
 test/inte/realtime_write_inte_test.cpp        | 258 +++++++++++++++++-
 3 files changed, 258 insertions(+), 13 deletions(-)

diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp
index f894e1a74..7f9fe568b 100644
--- a/src/paimon/core/table/source/table_scan.cpp
+++ b/src/paimon/core/table/source/table_scan.cpp
@@ -219,7 +219,7 @@ Result<std::unique_ptr<TableScan>> TableScan::Create(std::unique_ptr<ScanContext
 namespace {
 
 Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions& core_options,
-                            const ScanContext& context) {
+                            const ScanContext& context, bool read_optimized) {
     if (!context.GetRealtimeContext()) {
         return Status::OK();
     }
@@ -233,6 +233,10 @@ Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions&
         return Status::Invalid("real-time union read does not support data evolution");
     }
     if (!table_schema.PrimaryKeys().empty()) {
+        if (read_optimized) {
+            return Status::NotImplemented(
+                "PK real-time union read does not support read-optimized scans");
+        }
         PAIMON_RETURN_NOT_OK(ValidatePrimaryKeyRealtimeOptions(core_options, table_schema));
     }
     if (context.IsStreamingMode()) {
@@ -282,7 +286,8 @@ Result<std::unique_ptr<TableScan>> NewDataTableScan(const std::shared_ptr<ScanCo
                            CoreOptions::FromMap(options, context->GetSpecificFileSystem(), {}));
     core_options.WithCache(context->GetCache());
 
-    PAIMON_RETURN_NOT_OK(ValidateRealtimeScan(*table_schema, core_options, *context));
+    PAIMON_RETURN_NOT_OK(
+        ValidateRealtimeScan(*table_schema, core_options, *context, read_optimized));
     // validate options
     if (core_options.GetBucket() == -1) {
         if (!table_schema->PrimaryKeys().empty()) {
diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp
index 6abec946b..d7bfa0912 100644
--- a/src/paimon/core/table/system/read_optimized_system_table.cpp
+++ b/src/paimon/core/table/system/read_optimized_system_table.cpp
@@ -58,6 +58,10 @@ std::map<std::string, std::string> ReadOptimizedSystemTable::ReadOptimizedOption
 
 Result<std::unique_ptr<TableScan>> ReadOptimizedSystemTable::NewScan(
     const std::shared_ptr<ScanContext>& context) const {
+    if (context->GetRealtimeContext() && !table_schema_->PrimaryKeys().empty()) {
+        return Status::NotImplemented(
+            "PK real-time union read does not support read-optimized scans");
+    }
     auto options = ReadOptimizedOptions();
     ScanContextBuilder builder(table_path_);
     builder.SetOptions(options)
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index 86f8a4a54..f10e93858 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -58,6 +58,7 @@
 #include "paimon/defs.h"
 #include "paimon/file_store_commit.h"
 #include "paimon/file_store_write.h"
+#include "paimon/fs/file_system.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/orphan_files_cleaner.h"
 #include "paimon/predicate/function.h"
@@ -80,6 +81,33 @@
 namespace paimon::test {
 namespace {
 
+bool HasSuffix(const std::string& value, const std::string& suffix) {
+    return value.size() >= suffix.size() &&
+           value.compare(value.size() - suffix.size(), suffix.size(), suffix) == 0;
+}
+
+Result<std::set<std::string>> ListPhysicalArtifacts(const std::shared_ptr<FileSystem>& file_system,
+                                                    const std::string& root) {
+    std::set<std::string> artifacts;
+    std::vector<std::string> directories = {root};
+    while (!directories.empty()) {
+        std::string directory = std::move(directories.back());
+        directories.pop_back();
+        std::vector<BasicFileStatus> statuses;
+        PAIMON_RETURN_NOT_OK(file_system->ListDir(directory, &statuses));
+        for (const BasicFileStatus& status : statuses) {
+            if (status.IsDir()) {
+                directories.push_back(status.GetPath());
+            } else if (HasSuffix(status.GetPath(), ".orc") ||
+                       HasSuffix(status.GetPath(), ".index") ||
+                       HasSuffix(status.GetPath(), ".channel")) {
+                artifacts.insert(status.GetPath());
+            }
+        }
+    }
+    return artifacts;
+}
+
 class FailAllocationMemoryPool final : public MemoryPool {
  public:
     explicit FailAllocationMemoryPool(const std::shared_ptr<MemoryPool>& delegate)
@@ -426,6 +454,90 @@ class SplitCommitReaderRealtimeStore final : public DelegatingRealtimeStore {
     }
 };
 
+class FailAfterPhysicalFileBatchReader final : public BatchReader {
+ public:
+    FailAfterPhysicalFileBatchReader(std::unique_ptr<BatchReader> delegate,
+                                     const std::shared_ptr<FileSystem>& file_system,
+                                     std::string root, size_t baseline_artifact_count,
+                                     const std::shared_ptr<std::atomic<bool>>& saw_artifacts)
+        : delegate_(std::move(delegate)),
+          file_system_(file_system),
+          root_(std::move(root)),
+          baseline_artifact_count_(baseline_artifact_count),
+          saw_artifacts_(saw_artifacts) {}
+
+    Result<ReadBatch> NextBatch() override {
+        if (returned_batch_count_ < 4) {
+            ++returned_batch_count_;
+            return delegate_->NextBatch();
+        }
+        const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(30);
+        while (std::chrono::steady_clock::now() < deadline) {
+            PAIMON_ASSIGN_OR_RAISE(std::set<std::string> artifacts,
+                                   ListPhysicalArtifacts(file_system_, root_));
+            bool has_data = false;
+            for (const std::string& artifact : artifacts) {
+                has_data = has_data || HasSuffix(artifact, ".orc");
+            }
+            if (artifacts.size() > baseline_artifact_count_ && has_data) {
+                saw_artifacts_->store(true, std::memory_order_release);
+                return Status::IOError(
+                    "injected commit reader failure after physical file creation");
+            }
+            std::this_thread::sleep_for(std::chrono::milliseconds(1));
+        }
+        return Status::IOError("timed out waiting for partial physical files");
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return delegate_->GetReaderMetrics();
+    }
+
+    void Close() override {
+        delegate_->Close();
+    }
+
+ private:
+    std::unique_ptr<BatchReader> delegate_;
+    std::shared_ptr<FileSystem> file_system_;
+    std::string root_;
+    size_t baseline_artifact_count_;
+    std::shared_ptr<std::atomic<bool>> saw_artifacts_;
+    int32_t returned_batch_count_ = 0;
+};
+
+class FailAfterPhysicalFileRealtimeStore final : public DelegatingRealtimeStore {
+ public:
+    FailAfterPhysicalFileRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
+                                       const std::shared_ptr<FileSystem>& file_system,
+                                       const std::string& root, size_t baseline_artifact_count,
+                                       const std::shared_ptr<std::atomic<bool>>& saw_artifacts)
+        : DelegatingRealtimeStore(delegate),
+          file_system_(file_system),
+          root_(root),
+          baseline_artifact_count_(baseline_artifact_count),
+          saw_artifacts_(saw_artifacts) {}
+
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
+        const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
+        PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
+                               delegate_->CreateCommitReaders(segment));
+        if (readers.empty()) {
+            return Status::Invalid("commit reader failure test requires a reader");
+        }
+        readers[0] = std::make_unique<FailAfterPhysicalFileBatchReader>(
+            std::make_unique<SplitBatchReader>(std::move(readers[0])), file_system_, root_,
+            baseline_artifact_count_, saw_artifacts_);
+        return readers;
+    }
+
+ private:
+    std::shared_ptr<FileSystem> file_system_;
+    std::string root_;
+    size_t baseline_artifact_count_;
+    std::shared_ptr<std::atomic<bool>> saw_artifacts_;
+};
+
 enum class CommitReaderMalformation { DROP_LAST, UNSORTED, SUBSTITUTE_OFFSET };
 
 class CorruptingBatchReader final : public BatchReader {
@@ -1339,9 +1451,7 @@ class RealtimeWriteInteTest : public ::testing::Test {
                              TableRead::Create(std::move(read_context)));
         ASSERT_NOK_WITH_MSG(table_read->CreateReader(invalid_splits),
                             "unsupported real-time split version");
-        if (!primary_key) {
-            ASSERT_EQ(1, close_state->query_close_count->load(std::memory_order_acquire));
-        }
+        ASSERT_EQ(1, close_state->query_close_count->load(std::memory_order_acquire));
 
         std::vector<Row> expected_rows = p0_rows;
         expected_rows.insert(expected_rows.end(), p1_rows.begin(), p1_rows.end());
@@ -1510,6 +1620,33 @@ TEST_F(RealtimeWriteInteTest, TestPkRead) {
     ASSERT_TRUE(query_view->expired());
 }
 
+TEST_F(RealtimeWriteInteTest, TestPkRealtimeReadOptimizedScanUnsupported) {
+    CreatePkTable();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+    const std::vector<Row> rows = {{1, "one", "p0"}, {2, "two", "p0"}};
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         MakeBatch(rows, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, /*commit_identifier=*/0));
+    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
+    ASSERT_EQ(rows, disk_rows);
+
+    ScanContextBuilder scan_builder(table_path_ + "$ro");
+    scan_builder.SetOptions(options_).WithRealtimeContext(realtime_context).WithMemoryPool(pool_);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<ScanContext> scan_context, scan_builder.Finish());
+    Result<std::unique_ptr<TableScan>> scan = TableScan::Create(std::move(scan_context));
+    ASSERT_TRUE(scan.status().IsNotImplemented()) << scan.status().ToString();
+    ASSERT_NE(std::string::npos, scan.status().ToString().find(
+                                     "PK real-time union read does not support read-optimized"));
+    ASSERT_OK(writer->Close());
+}
+
 TEST_F(RealtimeWriteInteTest, TestPkDeleteInsertAndPinnedReadsAcrossRefresh) {
     CreatePkTable();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
@@ -2358,6 +2495,59 @@ TEST_F(RealtimeWriteInteTest, TestPkCommitReaderCloseFailure) {
     ASSERT_OK(writer->Close());
 }
 
+TEST_F(RealtimeWriteInteTest, TestPkPrepareFailureCleansPartialPhysicalFiles) {
+    options_[Options::WRITE_BATCH_SIZE] = "1";
+    options_[Options::TARGET_FILE_ROW_NUM] = "1";
+    options_["file-index.bitmap.columns"] = "payload";
+    options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B";
+    CreatePkTable();
+    std::shared_ptr<FileSystem> file_system = dir_->GetFileSystem();
+    ASSERT_OK_AND_ASSIGN(std::set<std::string> baseline_artifacts,
+                         ListPhysicalArtifacts(file_system, dir_->Str()));
+    auto saw_artifacts = std::make_shared<std::atomic<bool>>(false);
+    auto factory = MakeDecoratingFactory<FailAfterPhysicalFileRealtimeStore>(
+        file_system, dir_->Str(), baseline_artifacts.size(), saw_artifacts);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> failed_context,
+                         RealtimeContext::Create(factory));
+    WriteContextBuilder failed_builder(table_path_, commit_user_);
+    failed_builder.SetOptions(options_)
+        .WithStreamingMode(true)
+        .WithRealtimeContext(failed_context)
+        .WithTempDirectory(PathUtil::JoinPath(dir_->Str(), "tmp"));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> failed_write_context,
+                         failed_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> failed_writer,
+                         FileStoreWrite::Create(std::move(failed_write_context)));
+
+    const std::vector<Row> wal = {
+        {1, "old", "p0"}, {1, "new", "p0"}, {2, "two", "p0"}, {2, "gone", "p0"}};
+    const std::vector<RecordBatch::RowKind> row_kinds = {
+        RecordBatch::RowKind::INSERT, RecordBatch::RowKind::UPDATE_AFTER,
+        RecordBatch::RowKind::INSERT, RecordBatch::RowKind::DELETE};
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> failed_batch,
+                         MakeBatch(wal, /*partitioned=*/false, /*bucket=*/0, row_kinds));
+    ASSERT_OK(failed_writer->Write(std::move(failed_batch)));
+    Result<std::vector<RealtimeCommitProgress>> failed_prepare =
+        failed_writer->PrepareCommitWithProgress(/*commit_identifier=*/0);
+    ASSERT_TRUE(failed_prepare.status().IsIOError()) << failed_prepare.status().ToString();
+    ASSERT_NE(std::string::npos,
+              failed_prepare.status().ToString().find(
+                  "injected commit reader failure after physical file creation"));
+    ASSERT_TRUE(saw_artifacts->load(std::memory_order_acquire));
+    ASSERT_OK_AND_ASSIGN(std::set<std::string> artifacts_after_abort,
+                         ListPhysicalArtifacts(file_system, dir_->Str()));
+    ASSERT_EQ(baseline_artifacts, artifacts_after_abort);
+
+    ASSERT_OK(failed_writer->Close());
+    failed_writer.reset();
+    failed_context.reset();
+
+    const std::vector<Row> expected_rows = {{1, "new", "p0"}};
+    ReplayPkWalAndCommit(wal, row_kinds, /*commit_identifier=*/0, expected_rows);
+    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap committed_offsets, ReadCommittedOffsets());
+    ASSERT_EQ(4, committed_offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
+}
+
 TEST_F(RealtimeWriteInteTest, TestRollingFilesPreserveProgress) {
     options_[Options::TARGET_FILE_ROW_NUM] = "10";
     CreateTable(/*partition_keys=*/{});
@@ -3819,7 +4009,45 @@ void RealtimeWriteInteTest::RunConcurrencyTest(bool primary_key) {
     constexpr int32_t kReadThreadCount = 4;
     constexpr int64_t kBatchCount = 12;
     constexpr int64_t kRowsPerBatch = 2;
-    constexpr int64_t kTotalRows = kBatchCount * kRowsPerBatch;
+    const int64_t total_rows = kBatchCount * (primary_key ? 3 : kRowsPerBatch);
+
+    std::vector<std::vector<Row>> pk_batches;
+    std::vector<std::vector<RecordBatch::RowKind>> pk_row_kinds;
+    std::vector<std::vector<Row>> pk_expected_states(1);
+    if (primary_key) {
+        std::map<int64_t, Row> current_rows;
+        for (int64_t batch_index = 0; batch_index < kBatchCount; ++batch_index) {
+            const int64_t key = batch_index % 4;
+            const int64_t deleted_key = (key + 2) % 4;
+            std::vector<Row> rows = {{key, "update-" + std::to_string(batch_index), "p0"},
+                                     {key, "latest-" + std::to_string(batch_index), "p0"},
+                                     {deleted_key, "deleted-" + std::to_string(batch_index), "p0"}};
+            pk_batches.push_back(rows);
+            pk_row_kinds.push_back({batch_index < 4 ? RecordBatch::RowKind::INSERT
+                                                    : RecordBatch::RowKind::UPDATE_AFTER,
+                                    RecordBatch::RowKind::UPDATE_AFTER,
+                                    RecordBatch::RowKind::DELETE});
+            current_rows[key] = rows[1];
+            current_rows.erase(deleted_key);
+            std::vector<Row> expected;
+            for (const auto& [id, row] : current_rows) {
+                static_cast<void>(id);
+                expected.push_back(row);
+            }
+            pk_expected_states.push_back(std::move(expected));
+        }
+    }
+
+    auto validate_read = [&](const std::vector<Row>& rows) {
+        if (!primary_key) {
+            return ValidateReadPrefix(rows, total_rows);
+        }
+        if (std::find(pk_expected_states.begin(), pk_expected_states.end(), rows) ==
+            pk_expected_states.end()) {
+            return Status::Invalid("PK real-time read does not match any completed write");
+        }
+        return Status::OK();
+    };
 
     std::atomic<bool> writer_done{false};
     std::atomic<bool> prepare_done{false};
@@ -3856,10 +4084,14 @@ void RealtimeWriteInteTest::RunConcurrencyTest(bool primary_key) {
         state.WaitForStart();
         for (int64_t batch_index = 0; batch_index < kBatchCount && !state.ShouldStop();
              ++batch_index) {
-            std::vector<Row> rows =
-                MakeRows(batch_index * kRowsPerBatch, kRowsPerBatch, /*partition=*/"p0");
+            std::vector<Row> rows = primary_key
+                                        ? pk_batches[static_cast<size_t>(batch_index)]
+                                        : MakeRows(batch_index * kRowsPerBatch, kRowsPerBatch,
+                                                   /*partition=*/"p0");
             Result<std::unique_ptr<RecordBatch>> batch_result =
-                MakeBatch(rows, /*partitioned=*/false);
+                primary_key ? MakeBatch(rows, /*partitioned=*/false, /*bucket=*/0,
+                                        pk_row_kinds[static_cast<size_t>(batch_index)])
+                            : MakeBatch(rows, /*partitioned=*/false);
             if (state.RecordErrorIfNotOk(batch_result)) {
                 break;
             }
@@ -3994,7 +4226,7 @@ void RealtimeWriteInteTest::RunConcurrencyTest(bool primary_key) {
                 if (state.RecordErrorIfNotOk(result)) {
                     break;
                 }
-                Status status = ValidateReadPrefix(result.value(), kTotalRows);
+                Status status = validate_read(result.value());
                 if (state.RecordErrorIfNotOk(status)) {
                     break;
                 }
@@ -4036,10 +4268,14 @@ void RealtimeWriteInteTest::RunConcurrencyTest(bool primary_key) {
     ASSERT_GE(commit_count.load(), 2);
     ASSERT_GE(refresh_count.load(), 2);
     ASSERT_OK_AND_ASSIGN(std::vector<Row> final_rows, ReadRows(realtime_context));
-    ASSERT_EQ(kTotalRows, static_cast<int64_t>(final_rows.size()));
-    ASSERT_OK(ValidateReadPrefix(final_rows, kTotalRows));
+    if (primary_key) {
+        ASSERT_EQ(pk_expected_states.back(), final_rows);
+    } else {
+        ASSERT_EQ(total_rows, static_cast<int64_t>(final_rows.size()));
+        ASSERT_OK(ValidateReadPrefix(final_rows, total_rows));
+    }
     ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap committed_offsets, ReadCommittedOffsets());
-    ASSERT_EQ(kTotalRows,
+    ASSERT_EQ(total_rows,
               committed_offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
     ASSERT_OK_AND_ASSIGN(uint64_t memory_usage, GetRealtimeMemoryUsage(realtime_context));
     ASSERT_EQ(0, memory_usage);

From b187e07dc3ba056d4f46620a38f6e489d65c00a2 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Tue, 25 Aug 2026 14:17:59 +0800
Subject: [PATCH 29/62] fix(realtime): harden prepared store handling

---
 include/paimon/realtime/realtime_store.h      |  6 ++
 .../merged_key_value_record_reader_test.cpp   | 97 +++++++++++++------
 .../realtime/prepared_key_value_reader.cpp    | 20 +---
 .../core/realtime/prepared_key_value_reader.h | 10 +-
 .../core/realtime/realtime_context_impl.cpp   | 10 +-
 .../core/realtime/realtime_context_impl.h     |  4 +-
 .../core/realtime/realtime_context_test.cpp   | 31 ++++--
 .../realtime/realtime_primary_key_writer.cpp  | 12 ++-
 8 files changed, 119 insertions(+), 71 deletions(-)

diff --git a/include/paimon/realtime/realtime_store.h b/include/paimon/realtime/realtime_store.h
index 03ef279a3..9ed1e4361 100644
--- a/include/paimon/realtime/realtime_store.h
+++ b/include/paimon/realtime/realtime_store.h
@@ -55,15 +55,21 @@ struct PAIMON_EXPORT PrimaryKeyRealtimeStoreCreateConfig {
 using RealtimeStoreCreateConfig =
     std::variant<AppendRealtimeStoreCreateConfig, PrimaryKeyRealtimeStoreCreateConfig>;
 
+/// Parameters used by a `RealtimeStoreFactory` to create a store.
 struct PAIMON_EXPORT RealtimeStoreCreateRequest {
     /// Schema whose ownership is transferred to the factory. Append mode receives the complete
     /// table write schema. Primary-key mode receives the prepared transport schema:
     /// [_VALUE_KIND, _SEQUENCE_NUMBER, _REALTIME_OFFSET, table write fields].
     std::unique_ptr<::ArrowSchema> write_schema;
+    /// Table options available to the store implementation.
     std::map<std::string, std::string> options;
+    /// Memory pool for allocations retained by the store.
     std::shared_ptr<MemoryPool> memory_pool;
+    /// Partition values identifying the store.
     std::map<std::string, std::string> partition;
+    /// Bucket identifying the store within its partition.
     int32_t bucket = -1;
+    /// Mode-specific store configuration.
     RealtimeStoreCreateConfig mode_config;
 };
 
diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index 79217828c..81a1f133e 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -65,6 +65,23 @@ std::shared_ptr<arrow::Schema> MakePreparedSchema(const arrow::FieldVector& valu
     return arrow::schema(prepared_fields);
 }
 
+Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderForTest(
+    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::optional<OffsetRange>& visible_offsets,
+    const std::shared_ptr<arrow::Schema>& key_schema,
+    const std::shared_ptr<arrow::Schema>& value_schema,
+    const std::shared_ptr<MemoryPool>& memory_pool) {
+    if (!key_schema) {
+        return Status::Invalid("prepared key schema cannot be null");
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::vector<DataField> key_fields,
+                           DataField::ConvertArrowSchemaToDataFields(key_schema));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FieldsComparator> key_comparator,
+                           FieldsComparator::Create(key_fields, /*is_ascending_order=*/true));
+    return AdaptPreparedBatchReader(std::move(reader), prepared_schema, visible_offsets, key_schema,
+                                    value_schema, key_comparator, memory_pool);
+}
+
 class TrackingBatchReader : public BatchReader {
  public:
     TrackingBatchReader(std::unique_ptr<BatchReader>&& delegate, int32_t* close_count)
@@ -217,8 +234,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderOffsetFilter) {
     auto batch_reader = std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 2);
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, OffsetRange(2, 4),
-                                 key_schema, value_schema, pool_));
+        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, OffsetRange(2, 4),
+                                        key_schema, value_schema, pool_));
     ASSERT_OK_AND_ASSIGN(
         std::vector<KeyValue> results,
         (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
@@ -232,6 +249,23 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderOffsetFilter) {
     KeyValueChecker::CheckResult(expected, results, 1, 2);
 }
 
+TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderRejectsReversedVisibleOffsets) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1]])").ValueOrDie();
+    auto batch_reader =
+        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, /*batch_size=*/1);
+
+    Result<std::unique_ptr<KeyValueRecordReader>> result =
+        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, OffsetRange(2, 1),
+                                        value_schema, value_schema, pool_);
+    ASSERT_TRUE(result.status().IsInvalid());
+    ASSERT_NOK_WITH_MSG(result, "prepared visible offset range begin exceeds end");
+}
+
 TEST_F(MergedKeyValueRecordReaderTest, TestRejectsUnsortedPluginRowsAcrossBatches) {
     std::vector<DataField> value_fields = {DataField(0, arrow::field("id", arrow::int32()))};
     std::shared_ptr<arrow::Schema> value_schema =
@@ -247,9 +281,10 @@ TEST_F(MergedKeyValueRecordReaderTest, TestRejectsUnsortedPluginRowsAcrossBatche
             .ValueOrDie();
     auto batch_reader =
         std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, /*batch_size=*/1);
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
-                         AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema,
-                                                  std::nullopt, key_schema, value_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, std::nullopt,
+                                        key_schema, value_schema, pool_));
     Result<std::vector<KeyValue>> result =
         ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
                                                    KeyValueRecordReader::Iterator>(reader.get());
@@ -270,8 +305,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderCommitSchema) {
         std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> query_reader,
-        AdaptPreparedBatchReader(std::move(query_batch_reader), prepared_schema, OffsetRange(0, 1),
-                                 value_schema, value_schema, pool_));
+        AdaptPreparedBatchReaderForTest(std::move(query_batch_reader), prepared_schema,
+                                        OffsetRange(0, 1), value_schema, value_schema, pool_));
     ASSERT_OK_AND_ASSIGN(
         std::vector<KeyValue> query_results,
         (ReadResultCollector::CollectKeyValueResult<
@@ -281,9 +316,10 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderCommitSchema) {
     ASSERT_EQ(query_results[0].value->GetInt(0), 1);
 
     auto batch_reader = std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
-    ASSERT_NOK_WITH_MSG(AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema,
-                                                 std::nullopt, value_schema, value_schema, pool_),
-                        "exact");
+    ASSERT_NOK_WITH_MSG(
+        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, std::nullopt,
+                                        value_schema, value_schema, pool_),
+        "exact");
 }
 
 TEST_F(MergedKeyValueRecordReaderTest, TestBadCommitBatch) {
@@ -299,8 +335,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestBadCommitBatch) {
     auto batch_reader = std::make_unique<MockFileBatchReader>(actual, actual_type, 1);
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, std::nullopt,
-                                 arrow::schema({key}), value_schema, pool_));
+        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, std::nullopt,
+                                        arrow::schema({key}), value_schema, pool_));
     ASSERT_NOK_WITH_MSG(reader->NextBatch(), "field count");
 }
 
@@ -319,8 +355,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestMissingCompositeKey) {
     auto batch_reader = std::make_unique<MockFileBatchReader>(actual, actual_type, 1);
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
-                                 arrow::schema({key0, key1}), value_schema, pool_));
+        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
+                                        arrow::schema({key0, key1}), value_schema, pool_));
     ASSERT_NOK_WITH_MSG(reader->NextBatch(), "cannot find field id 1");
 }
 
@@ -341,8 +377,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestQueryAddRename) {
     auto batch_reader = std::make_unique<MockFileBatchReader>(actual, actual_type, 1);
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
-                                 arrow::schema({key}), value_schema, pool_));
+        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
+                                        arrow::schema({key}), value_schema, pool_));
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader::Iterator> iterator,
                          reader->NextBatch());
     ASSERT_OK_AND_ASSIGN(KeyValue key_value, iterator->Next());
@@ -361,8 +397,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestMergedReaderErrorRetry) {
     failing_reader->SetNextBatchStatus(Status::IOError("stable prepared error"));
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReader(std::move(failing_reader), prepared_schema, OffsetRange(0, 1),
-                                 value_schema, value_schema, pool_));
+        AdaptPreparedBatchReaderForTest(std::move(failing_reader), prepared_schema,
+                                        OffsetRange(0, 1), value_schema, value_schema, pool_));
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<FieldsComparator> key_comparator,
                          FieldsComparator::Create({DataField(0, key)}, true));
     MergedKeyValueRecordReader merged_reader(std::move(reader), key_comparator,
@@ -390,8 +426,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderSafeDecode) {
     auto batch_reader = std::make_unique<MockFileBatchReader>(invalid_array, invalid_type, 1);
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
-                                 value_schema, value_schema, pool_));
+        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
+                                        value_schema, value_schema, pool_));
     ASSERT_NOK_WITH_MSG(
         (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
                                                     KeyValueRecordReader::Iterator>(reader.get())),
@@ -448,8 +484,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderNestedProjection) {
     auto batch_reader = std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
-                                 key_schema, query_value_schema, pool_));
+        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
+                                        key_schema, query_value_schema, pool_));
     ASSERT_OK_AND_ASSIGN(
         std::vector<KeyValue> results,
         (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
@@ -516,8 +552,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderLifecycle) {
             &explicit_close_count);
         ASSERT_OK_AND_ASSIGN(
             std::unique_ptr<KeyValueRecordReader> reader,
-            AdaptPreparedBatchReader(std::move(tracking_reader), prepared_schema, OffsetRange(0, 1),
-                                     key_schema, value_schema, pool_));
+            AdaptPreparedBatchReaderForTest(std::move(tracking_reader), prepared_schema,
+                                            OffsetRange(0, 1), key_schema, value_schema, pool_));
         reader->Close();
     }
     ASSERT_EQ(explicit_close_count, 1);
@@ -529,8 +565,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderLifecycle) {
             &destructor_close_count);
         ASSERT_OK_AND_ASSIGN(
             std::unique_ptr<KeyValueRecordReader> reader,
-            AdaptPreparedBatchReader(std::move(tracking_reader), prepared_schema, OffsetRange(0, 1),
-                                     key_schema, value_schema, pool_));
+            AdaptPreparedBatchReaderForTest(std::move(tracking_reader), prepared_schema,
+                                            OffsetRange(0, 1), key_schema, value_schema, pool_));
     }
     ASSERT_EQ(destructor_close_count, 1);
 
@@ -540,8 +576,9 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderLifecycle) {
             std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1),
             &factory_failure_close_count);
         std::shared_ptr<arrow::Schema> invalid_schema = arrow::schema(value_schema->fields());
-        ASSERT_NOK(AdaptPreparedBatchReader(std::move(tracking_reader), invalid_schema,
-                                            OffsetRange(0, 1), key_schema, value_schema, pool_));
+        ASSERT_NOK(AdaptPreparedBatchReaderForTest(std::move(tracking_reader), invalid_schema,
+                                                   OffsetRange(0, 1), key_schema, value_schema,
+                                                   pool_));
         ASSERT_EQ(nullptr, tracking_reader);
     }
     ASSERT_EQ(factory_failure_close_count, 1);
@@ -555,8 +592,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderLifecycle) {
                                                                      &read_failure_close_count);
         ASSERT_OK_AND_ASSIGN(
             std::unique_ptr<KeyValueRecordReader> reader,
-            AdaptPreparedBatchReader(std::move(tracking_reader), prepared_schema, OffsetRange(0, 1),
-                                     key_schema, value_schema, pool_));
+            AdaptPreparedBatchReaderForTest(std::move(tracking_reader), prepared_schema,
+                                            OffsetRange(0, 1), key_schema, value_schema, pool_));
         ASSERT_NOK_WITH_MSG(reader->NextBatch(), "prepared reader failure");
         ASSERT_EQ(read_failure_close_count, 1);
     }
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index 623e3f3fd..fa1dba7ef 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -690,6 +690,9 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderImpl(
         return Status::Invalid("prepared batch reader cannot be null");
     }
     ScopeGuard close_guard([&owned_reader]() -> void { owned_reader->Close(); });
+    if (visible_offsets.has_value() && visible_offsets->begin > visible_offsets->end) {
+        return Status::Invalid("prepared visible offset range begin exceeds end");
+    }
     PAIMON_RETURN_NOT_OK(ValidatePreparedTransportSchema(prepared_schema));
     if (!key_schema) {
         return Status::Invalid("prepared key schema cannot be null");
@@ -765,21 +768,4 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> AdaptPreparedCommitBa
     return adapted_readers;
 }
 
-Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
-    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
-    const std::optional<OffsetRange>& visible_offsets,
-    const std::shared_ptr<arrow::Schema>& key_schema,
-    const std::shared_ptr<arrow::Schema>& value_schema,
-    const std::shared_ptr<MemoryPool>& memory_pool) {
-    if (!key_schema) {
-        return Status::Invalid("prepared key schema cannot be null");
-    }
-    PAIMON_ASSIGN_OR_RAISE(std::vector<DataField> key_fields,
-                           DataField::ConvertArrowSchemaToDataFields(key_schema));
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FieldsComparator> key_comparator,
-                           FieldsComparator::Create(key_fields, /*is_ascending_order=*/true));
-    return AdaptPreparedBatchReader(std::move(reader), prepared_schema, visible_offsets, key_schema,
-                                    value_schema, key_comparator, memory_pool);
-}
-
 }  // namespace paimon
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.h b/src/paimon/core/realtime/prepared_key_value_reader.h
index 22a837a76..81ae0abc7 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.h
+++ b/src/paimon/core/realtime/prepared_key_value_reader.h
@@ -33,8 +33,10 @@ class BatchReader;
 class FieldsComparator;
 class MemoryPool;
 
+/// Validates the required leading fields of a prepared real-time transport schema.
 Status ValidatePreparedTransportSchema(const std::shared_ptr<arrow::Schema>& prepared_schema);
 
+/// Adapts a plugin query reader and limits its rows to `visible_offsets` when present.
 Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
     std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
     const std::optional<OffsetRange>& visible_offsets,
@@ -43,6 +45,7 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
     const std::shared_ptr<FieldsComparator>& key_comparator,
     const std::shared_ptr<MemoryPool>& memory_pool);
 
+/// Adapts commit readers and validates that they collectively cover `sealed_offsets` exactly.
 Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> AdaptPreparedCommitBatchReaders(
     std::vector<std::unique_ptr<BatchReader>>&& readers,
     const std::shared_ptr<arrow::Schema>& prepared_schema, const OffsetRange& sealed_offsets,
@@ -51,11 +54,4 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> AdaptPreparedCommitBa
     const std::shared_ptr<FieldsComparator>& key_comparator,
     const std::shared_ptr<MemoryPool>& memory_pool);
 
-Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
-    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
-    const std::optional<OffsetRange>& visible_offsets,
-    const std::shared_ptr<arrow::Schema>& key_schema,
-    const std::shared_ptr<arrow::Schema>& value_schema,
-    const std::shared_ptr<MemoryPool>& memory_pool);
-
 }  // namespace paimon
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index 9b12aa5bd..c9f719b7e 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -168,10 +168,16 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     return RealtimeStoreState{std::move(store), initial_offset};
 }
 
-int64_t RealtimeContextImpl::AdvanceMaterializedMaxSequenceNumber(
+Result<int64_t> RealtimeContextImpl::AdvanceMaterializedMaxSequenceNumber(
     const RealtimePartitionBucket& partition_bucket, int64_t max_sequence_number) {
     std::lock_guard<std::mutex> lock(mutex_);
-    StoreEntry& entry = stores_.at(partition_bucket);
+    auto iter = stores_.find(partition_bucket);
+    if (iter == stores_.end()) {
+        return Status::KeyError("real-time store not found for partition " +
+                                PartitionToString(partition_bucket.partition) + ", bucket " +
+                                std::to_string(partition_bucket.bucket));
+    }
+    StoreEntry& entry = iter->second;
     if (max_sequence_number > entry.materialized_max_sequence_number) {
         entry.materialized_max_sequence_number = max_sequence_number;
     }
diff --git a/src/paimon/core/realtime/realtime_context_impl.h b/src/paimon/core/realtime/realtime_context_impl.h
index f0014176d..fd65fc246 100644
--- a/src/paimon/core/realtime/realtime_context_impl.h
+++ b/src/paimon/core/realtime/realtime_context_impl.h
@@ -71,8 +71,8 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
 
     Result<RealtimeStoreState> GetOrCreateRealtimeStore(RealtimeStoreCreateRequest&& request);
 
-    int64_t AdvanceMaterializedMaxSequenceNumber(const RealtimePartitionBucket& partition_bucket,
-                                                 int64_t max_sequence_number);
+    Result<int64_t> AdvanceMaterializedMaxSequenceNumber(
+        const RealtimePartitionBucket& partition_bucket, int64_t max_sequence_number);
 
     Result<std::vector<RealtimePartitionBucketView>> AcquireReadViews();
 
diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp
index 5dc5d8b4f..538e4c56c 100644
--- a/src/paimon/core/realtime/realtime_context_test.cpp
+++ b/src/paimon/core/realtime/realtime_context_test.cpp
@@ -192,14 +192,29 @@ TEST(RealtimeContextTest, TestReconcilesPrimaryKeyInitialSequence) {
     ASSERT_OK(
         GetOrCreateAppendStore(context, partition, 0, MakeWriteSchema(), {}, GetDefaultPool()));
 
-    ASSERT_EQ(4, context->AdvanceMaterializedMaxSequenceNumber(partition_bucket,
-                                                               /*max_sequence_number=*/4));
-    ASSERT_EQ(8, context->AdvanceMaterializedMaxSequenceNumber(partition_bucket,
-                                                               /*max_sequence_number=*/8));
-    ASSERT_EQ(8, context->AdvanceMaterializedMaxSequenceNumber(partition_bucket,
-                                                               /*max_sequence_number=*/6));
-    ASSERT_EQ(10, context->AdvanceMaterializedMaxSequenceNumber(partition_bucket,
-                                                                /*max_sequence_number=*/10));
+    ASSERT_OK_AND_ASSIGN(int64_t first, context->AdvanceMaterializedMaxSequenceNumber(
+                                            partition_bucket, /*max_sequence_number=*/4));
+    ASSERT_EQ(4, first);
+    ASSERT_OK_AND_ASSIGN(int64_t second, context->AdvanceMaterializedMaxSequenceNumber(
+                                             partition_bucket, /*max_sequence_number=*/8));
+    ASSERT_EQ(8, second);
+    ASSERT_OK_AND_ASSIGN(int64_t third, context->AdvanceMaterializedMaxSequenceNumber(
+                                            partition_bucket, /*max_sequence_number=*/6));
+    ASSERT_EQ(8, third);
+    ASSERT_OK_AND_ASSIGN(int64_t fourth, context->AdvanceMaterializedMaxSequenceNumber(
+                                             partition_bucket, /*max_sequence_number=*/10));
+    ASSERT_EQ(10, fourth);
+}
+
+TEST(RealtimeContextTest, TestMaterializedSequenceRejectsMissingStore) {
+    auto factory = std::make_shared<TestingRealtimeStoreFactory>();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
+
+    Result<int64_t> result = context->AdvanceMaterializedMaxSequenceNumber(
+        RealtimePartitionBucket({{"dt", "missing"}}, /*bucket=*/3),
+        /*max_sequence_number=*/4);
+    ASSERT_TRUE(result.status().IsKeyError());
+    ASSERT_NOK_WITH_MSG(result, "real-time store not found for partition {dt=missing}, bucket 3");
 }
 
 TEST(RealtimeContextTest, TestCommittedProgressIsMonotonicAndSelective) {
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index 7f4d4b5f7..b04cc8e25 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -172,9 +172,9 @@ Result<std::shared_ptr<RealtimePrimaryKeyWriter>> RealtimePrimaryKeyWriter::Crea
     prepared_fields.insert(prepared_fields.end(), write_schema->fields().begin(),
                            write_schema->fields().end());
     const RealtimePartitionBucket partition_bucket(partition, bucket);
-    const int64_t initial_max_sequence_number =
-        realtime_context->AdvanceMaterializedMaxSequenceNumber(partition_bucket,
-                                                               restored_max_sequence_number);
+    PAIMON_ASSIGN_OR_RAISE(int64_t initial_max_sequence_number,
+                           realtime_context->AdvanceMaterializedMaxSequenceNumber(
+                               partition_bucket, restored_max_sequence_number));
     return std::shared_ptr<RealtimePrimaryKeyWriter>(new RealtimePrimaryKeyWriter(
         store_state.store, merge_tree_writer, realtime_context, partition_bucket, write_schema,
         arrow::schema(std::move(prepared_fields)), arrow::schema(std::move(key_fields)),
@@ -246,8 +246,10 @@ Status RealtimePrimaryKeyWriter::Write(std::unique_ptr<RecordBatch>&& batch) {
         std::move(prepared_batch), OffsetRange(next_offset_, next_offset_ + count)}));
     next_offset_ += count;
     last_sequence_number_ += count;
-    realtime_context_->AdvanceMaterializedMaxSequenceNumber(partition_bucket_,
-                                                            last_sequence_number_);
+    PAIMON_RETURN_NOT_OK(
+        realtime_context_
+            ->AdvanceMaterializedMaxSequenceNumber(partition_bucket_, last_sequence_number_)
+            .status());
     return Status::OK();
 }
 

From 5a6dcb657515ea35570b012d24bcd279faeb3ea3 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Tue, 25 Aug 2026 15:56:16 +0800
Subject: [PATCH 30/62] refactor(realtime): simplify PK store boundary

---
 include/paimon/realtime/realtime_store.h      |   5 +-
 .../operation/key_value_file_store_write.cpp  |   2 +-
 .../realtime/arrow_realtime_store_factory.cpp |   5 +-
 .../realtime/prepared_key_value_reader.cpp    | 102 ++++-----
 .../realtime/primary_key_realtime_store.cpp   | 215 ++----------------
 .../realtime/primary_key_realtime_store.h     |   5 +-
 .../primary_key_realtime_store_test.cpp       | 131 +++--------
 .../core/realtime/realtime_context_impl.cpp   |   9 +-
 test/inte/realtime_write_inte_test.cpp        |  25 +-
 9 files changed, 111 insertions(+), 388 deletions(-)

diff --git a/include/paimon/realtime/realtime_store.h b/include/paimon/realtime/realtime_store.h
index 9ed1e4361..e61051e29 100644
--- a/include/paimon/realtime/realtime_store.h
+++ b/include/paimon/realtime/realtime_store.h
@@ -47,10 +47,7 @@ struct PAIMON_EXPORT AppendRealtimeStoreCreateConfig {
     StatisticsMode statistics_mode;
 };
 
-struct PAIMON_EXPORT PrimaryKeyRealtimeStoreCreateConfig {
-    /// Primary-key fields after removing partition fields, in comparison order.
-    std::vector<std::string> trimmed_primary_keys;
-};
+struct PAIMON_EXPORT PrimaryKeyRealtimeStoreCreateConfig {};
 
 using RealtimeStoreCreateConfig =
     std::variant<AppendRealtimeStoreCreateConfig, PrimaryKeyRealtimeStoreCreateConfig>;
diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp
index 737ece080..31889ee84 100644
--- a/src/paimon/core/operation/key_value_file_store_write.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write.cpp
@@ -154,7 +154,7 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
             RealtimeStoreState store_state,
             realtime_context_impl->GetOrCreateRealtimeStore(RealtimeStoreCreateRequest{
                 std::move(c_write_schema), options_.ToMap(), pool_, partition_map, bucket,
-                PrimaryKeyRealtimeStoreCreateConfig{trimmed_primary_keys}}));
+                PrimaryKeyRealtimeStoreCreateConfig{}}));
         realtime_store_state = std::move(store_state);
         compact_manager = std::make_shared<NoopCompactManager>();
     } else {
diff --git a/src/paimon/core/realtime/arrow_realtime_store_factory.cpp b/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
index babc55a3d..d336394ad 100644
--- a/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
+++ b/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
@@ -50,11 +50,8 @@ Result<std::shared_ptr<RealtimeStore>> ArrowRealtimeStoreFactory::Create(
                                                     request.memory_pool, arrow_pool);
     }
 
-    const PrimaryKeyRealtimeStoreCreateConfig& config =
-        std::get<PrimaryKeyRealtimeStoreCreateConfig>(request.mode_config);
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                           PrimaryKeyRealtimeStore::Create(
-                               imported_schema, config.trimmed_primary_keys, request.memory_pool));
+                           PrimaryKeyRealtimeStore::Create(imported_schema));
     return std::shared_ptr<RealtimeStore>(std::move(store));
 }
 
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index fa1dba7ef..c17513aef 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -32,7 +32,6 @@
 #include "arrow/array/builder_primitive.h"
 #include "arrow/buffer.h"
 #include "arrow/c/bridge.h"
-#include "arrow/compute/api.h"
 #include "arrow/type.h"
 #include "arrow/util/bit_util.h"
 #include "fmt/format.h"
@@ -386,39 +385,6 @@ Result<arrow::ArrayVector> ProjectFieldsByPaimonIds(
     return result;
 }
 
-Result<std::shared_ptr<arrow::StructArray>> ApplyOffsetFilter(
-    const std::shared_ptr<arrow::StructArray>& data_batch,
-    const std::shared_ptr<arrow::NumericArray<arrow::Int64Type>>& offset_array,
-    const std::optional<OffsetRange>& visible_offsets, arrow::MemoryPool* arrow_pool) {
-    if (!visible_offsets.has_value()) {
-        return data_batch;
-    }
-
-    arrow::BooleanBuilder filter_builder(arrow_pool);
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(filter_builder.Reserve(offset_array->length()));
-    int64_t visible_row_count = 0;
-    for (int64_t i = 0; i < offset_array->length(); ++i) {
-        int64_t offset = offset_array->Value(i);
-        bool visible = offset >= visible_offsets->begin && offset < visible_offsets->end;
-        filter_builder.UnsafeAppend(visible);
-        visible_row_count += visible;
-    }
-    if (visible_row_count == 0) {
-        return std::shared_ptr<arrow::StructArray>();
-    }
-    if (visible_row_count == data_batch->length()) {
-        return data_batch;
-    }
-    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> filter,
-                                      filter_builder.Finish());
-    arrow::compute::ExecContext exec_context(arrow_pool);
-    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-        arrow::Datum filtered,
-        arrow::compute::Filter(data_batch, filter, arrow::compute::FilterOptions::Defaults(),
-                               &exec_context));
-    return checked_pointer_cast<arrow::StructArray>(filtered.make_array());
-}
-
 class PreparedKeyValueReader final : public KeyValueRecordReader {
  public:
     PreparedKeyValueReader(std::unique_ptr<BatchReader>&& reader,
@@ -448,20 +414,20 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
         explicit Iterator(PreparedKeyValueReader* reader) : reader_(reader) {}
 
         Result<bool> HasNext() const override {
-            return cursor_ < reader_->row_kind_array_->length();
+            return cursor_ < reader_->RowCount();
         }
 
         Result<KeyValue> Next() override {
-            if (cursor_ >= reader_->row_kind_array_->length()) {
+            if (cursor_ >= reader_->RowCount()) {
                 return Status::Invalid("No more prepared key values in current iterator");
             }
+            const int64_t row = reader_->RowAt(cursor_);
             std::shared_ptr<InternalRow> key =
-                std::make_shared<ColumnarRowRef>(reader_->key_ctx_, cursor_);
-            auto value = std::make_unique<ColumnarRowRef>(reader_->value_ctx_, cursor_);
-            PAIMON_ASSIGN_OR_RAISE(
-                const RowKind* row_kind,
-                RowKind::FromByteValue(reader_->row_kind_array_->Value(cursor_)));
-            int64_t sequence_number = reader_->sequence_number_array_->Value(cursor_);
+                std::make_shared<ColumnarRowRef>(reader_->key_ctx_, row);
+            auto value = std::make_unique<ColumnarRowRef>(reader_->value_ctx_, row);
+            PAIMON_ASSIGN_OR_RAISE(const RowKind* row_kind,
+                                   RowKind::FromByteValue(reader_->row_kind_array_->Value(row)));
+            int64_t sequence_number = reader_->sequence_number_array_->Value(row);
             ++cursor_;
             return KeyValue(row_kind, sequence_number, KeyValue::UNKNOWN_LEVEL, std::move(key),
                             std::move(value));
@@ -535,7 +501,6 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
                 data_batch = checked_pointer_cast<arrow::StructArray>(arrow_array);
             }
             PAIMON_RETURN_NOT_OK(ValidatePreparedBatch(data_batch));
-            PAIMON_RETURN_NOT_OK(ValidateOrdering(data_batch));
 
             std::shared_ptr<arrow::NumericArray<arrow::Int64Type>> offset_array =
                 checked_pointer_cast<arrow::NumericArray<arrow::Int64Type>>(
@@ -543,12 +508,6 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
             if (offset_coverage_) {
                 PAIMON_RETURN_NOT_OK(offset_coverage_->Add(*offset_array));
             }
-            PAIMON_ASSIGN_OR_RAISE(
-                data_batch,
-                ApplyOffsetFilter(data_batch, offset_array, visible_offsets_, arrow_pool_.get()));
-            if (!data_batch) {
-                continue;
-            }
 
             row_kind_array_ = checked_pointer_cast<arrow::NumericArray<arrow::Int8Type>>(
                 data_batch->field(kValueKindIndex));
@@ -562,6 +521,10 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
                                                             value_schema_, arrow_pool_.get()));
             key_ctx_ = std::make_shared<ColumnarBatchContext>(key_fields, pool_);
             value_ctx_ = std::make_shared<ColumnarBatchContext>(value_fields, pool_);
+            PAIMON_RETURN_NOT_OK(ValidateOrdering(key_ctx_, sequence_number_array_));
+            if (!SelectVisibleRows(*offset_array)) {
+                continue;
+            }
             ArrowUtils::TraverseArray(data_batch);
             return std::make_unique<Iterator>(this);
         }
@@ -600,18 +563,13 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
         return Status::OK();
     }
 
-    Status ValidateOrdering(const std::shared_ptr<arrow::StructArray>& data_batch) {
-        if (data_batch->length() == 0) {
+    Status ValidateOrdering(
+        const std::shared_ptr<ColumnarBatchContext>& key_context,
+        const std::shared_ptr<arrow::NumericArray<arrow::Int64Type>>& sequences) {
+        if (sequences->length() == 0) {
             return Status::OK();
         }
-        PAIMON_ASSIGN_OR_RAISE(
-            arrow::ArrayVector key_fields,
-            ProjectFieldsByPaimonIds(data_batch, prepared_schema_, key_schema_, arrow_pool_.get()));
-        std::shared_ptr<ColumnarBatchContext> key_context =
-            std::make_shared<ColumnarBatchContext>(key_fields, pool_);
-        std::shared_ptr<arrow::Int64Array> sequences =
-            checked_pointer_cast<arrow::Int64Array>(data_batch->field(kSequenceNumberIndex));
-        for (int64_t row = 0; row < data_batch->length(); ++row) {
+        for (int64_t row = 0; row < sequences->length(); ++row) {
             ColumnarRowRef current_key(key_context, row);
             if (previous_key_context_) {
                 ColumnarRowRef previous_key(previous_key_context_, previous_key_row_);
@@ -631,11 +589,36 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
         return Status::OK();
     }
 
+    bool SelectVisibleRows(const arrow::Int64Array& offsets) {
+        if (!visible_offsets_.has_value()) {
+            return true;
+        }
+        visible_rows_.emplace();
+        visible_rows_->reserve(offsets.length());
+        for (int64_t row = 0; row < offsets.length(); ++row) {
+            const int64_t offset = offsets.Value(row);
+            if (offset >= visible_offsets_->begin && offset < visible_offsets_->end) {
+                visible_rows_->push_back(row);
+            }
+        }
+        return !visible_rows_->empty();
+    }
+
+    int64_t RowCount() const {
+        return visible_rows_.has_value() ? static_cast<int64_t>(visible_rows_->size())
+                                         : row_kind_array_->length();
+    }
+
+    int64_t RowAt(int64_t ordinal) const {
+        return visible_rows_.has_value() ? (*visible_rows_)[ordinal] : ordinal;
+    }
+
     void ResetBatchState() {
         key_ctx_.reset();
         value_ctx_.reset();
         row_kind_array_.reset();
         sequence_number_array_.reset();
+        visible_rows_.reset();
     }
 
  private:
@@ -655,6 +638,7 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
     std::shared_ptr<ColumnarBatchContext> value_ctx_;
     std::shared_ptr<arrow::NumericArray<arrow::Int8Type>> row_kind_array_;
     std::shared_ptr<arrow::NumericArray<arrow::Int64Type>> sequence_number_array_;
+    std::optional<std::vector<int64_t>> visible_rows_;
     std::shared_ptr<ColumnarBatchContext> previous_key_context_;
     int64_t previous_key_row_ = 0;
     int64_t previous_sequence_ = 0;
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index 22fab1b08..68f4bf500 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -18,25 +18,17 @@
 
 #include "paimon/core/realtime/primary_key_realtime_store.h"
 
-#include <cstddef>
 #include <mutex>
 #include <optional>
-#include <queue>
-#include <unordered_map>
 #include <utility>
 #include <vector>
 
 #include "arrow/api.h"
 #include "arrow/c/bridge.h"
-#include "arrow/compute/api.h"
-#include "paimon/common/data/columnar/columnar_batch_context.h"
-#include "paimon/common/data/columnar/columnar_row_ref.h"
 #include "paimon/common/metrics/metrics_impl.h"
 #include "paimon/common/types/data_field.h"
-#include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
-#include "paimon/common/utils/fields_comparator.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/index/pk/primary_key_index_definitions.h"
 #include "paimon/core/realtime/prepared_key_value_reader.h"
@@ -155,116 +147,19 @@ class ReadView final : public RealtimeReadView {
     std::optional<OffsetRange> range_;
 };
 
-class RawBatchReader final : public BatchReader {
+class StoredBatchReader final : public BatchReader {
  public:
-    RawBatchReader(std::vector<StoredBatch> batches, std::vector<int32_t> key_field_indexes,
-                   const std::shared_ptr<FieldsComparator>& key_comparator,
-                   const std::shared_ptr<MemoryPool>& memory_pool)
-        : batches_(std::move(batches)),
-          positions_(batches_.size(), 0),
-          key_field_indexes_(std::move(key_field_indexes)),
-          key_comparator_(key_comparator),
-          memory_pool_(memory_pool),
-          arrow_pool_(GetArrowPool(memory_pool)),
-          heap_(SourceGreater{this}),
-          metrics_(std::make_shared<MetricsImpl>()) {
-        key_contexts_.reserve(batches_.size());
-        sequence_arrays_.reserve(batches_.size());
-        for (size_t i = 0; i < batches_.size(); ++i) {
-            const StoredBatch& batch = batches_[i];
-            arrow::ArrayVector key_arrays;
-            key_arrays.reserve(key_field_indexes_.size());
-            for (int32_t field_index : key_field_indexes_) {
-                key_arrays.push_back(batch.data->field(field_index));
-            }
-            key_contexts_.push_back(
-                std::make_shared<ColumnarBatchContext>(key_arrays, memory_pool_));
-            sequence_arrays_.push_back(
-                checked_pointer_cast<arrow::Int64Array>(batch.data->field(1)));
-            if (batch.data->length() > 0) {
-                heap_.push(i);
-            }
-        }
-    }
+    explicit StoredBatchReader(const StoredBatch& batch)
+        : data_(batch.data), metrics_(std::make_shared<MetricsImpl>()) {}
 
     Result<ReadBatch> NextBatch() override {
-        if (heap_.empty()) {
+        if (!data_) {
             return MakeEofBatch();
         }
-
-        struct SelectedRow {
-            size_t selected_source;
-            int64_t source_ordinal;
-        };
-        struct SelectedSource {
-            size_t source;
-            std::vector<int64_t> rows;
-            int64_t base = -1;
-        };
-        std::vector<SelectedRow> selected_rows;
-        selected_rows.reserve(kOutputBatchSize);
-        std::vector<SelectedSource> selected_sources;
-        std::unordered_map<size_t, size_t> selected_source_indexes;
-        while (!heap_.empty() && selected_rows.size() < kOutputBatchSize) {
-            const size_t source = heap_.top();
-            heap_.pop();
-            auto [source_it, inserted] =
-                selected_source_indexes.emplace(source, selected_sources.size());
-            if (inserted) {
-                selected_sources.push_back(SelectedSource{source, {}});
-            }
-            SelectedSource& selected_source = selected_sources[source_it->second];
-            selected_rows.push_back(
-                SelectedRow{source_it->second, static_cast<int64_t>(selected_source.rows.size())});
-            selected_source.rows.push_back(positions_[source]++);
-            if (positions_[source] < batches_[source].data->length()) {
-                heap_.push(source);
-            }
-        }
-
-        arrow::compute::ExecContext context(arrow_pool_.get());
-        arrow::ArrayVector grouped_batches;
-        int64_t grouped_row_count = 0;
-        for (SelectedSource& selected_source : selected_sources) {
-            arrow::Int64Builder source_index_builder(arrow_pool_.get());
-            PAIMON_RETURN_NOT_OK_FROM_ARROW(
-                source_index_builder.AppendValues(selected_source.rows));
-            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> source_indices,
-                                              source_index_builder.Finish());
-            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-                arrow::Datum source_batch,
-                arrow::compute::Take(arrow::Datum(batches_[selected_source.source].data),
-                                     arrow::Datum(source_indices),
-                                     arrow::compute::TakeOptions::NoBoundsCheck(), &context));
-            selected_source.base = grouped_row_count;
-            grouped_row_count += static_cast<int64_t>(selected_source.rows.size());
-            grouped_batches.push_back(source_batch.make_array());
-        }
-
-        std::shared_ptr<arrow::Array> batch;
-        if (grouped_batches.size() == 1) {
-            batch = std::move(grouped_batches[0]);
-        } else {
-            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-                std::shared_ptr<arrow::Array> grouped,
-                arrow::Concatenate(grouped_batches, arrow_pool_.get()));
-            arrow::Int64Builder order_builder(arrow_pool_.get());
-            PAIMON_RETURN_NOT_OK_FROM_ARROW(order_builder.Reserve(selected_rows.size()));
-            for (const SelectedRow& selected : selected_rows) {
-                order_builder.UnsafeAppend(selected_sources[selected.selected_source].base +
-                                           selected.source_ordinal);
-            }
-            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> order,
-                                              order_builder.Finish());
-            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-                arrow::Datum reordered,
-                arrow::compute::Take(arrow::Datum(grouped), arrow::Datum(order),
-                                     arrow::compute::TakeOptions::NoBoundsCheck(), &context));
-            batch = reordered.make_array();
-        }
         auto array = std::make_unique<ArrowArray>();
         auto schema = std::make_unique<ArrowSchema>();
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*batch, array.get(), schema.get()));
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*data_, array.get(), schema.get()));
+        data_.reset();
         return ReadBatch(std::move(array), std::move(schema));
     }
 
@@ -272,50 +167,11 @@ class RawBatchReader final : public BatchReader {
         return metrics_;
     }
     void Close() override {
-        while (!heap_.empty()) {
-            heap_.pop();
-        }
-        batches_.clear();
-        positions_.clear();
-        key_contexts_.clear();
-        sequence_arrays_.clear();
+        data_.reset();
     }
 
  private:
-    static constexpr size_t kOutputBatchSize = 1024;
-
-    bool Less(size_t left, size_t right) const {
-        ColumnarRowRef left_key(key_contexts_[left], positions_[left]);
-        ColumnarRowRef right_key(key_contexts_[right], positions_[right]);
-        const int32_t key_comparison = key_comparator_->CompareTo(left_key, right_key);
-        if (key_comparison != 0) {
-            return key_comparison < 0;
-        }
-        const int64_t left_sequence = sequence_arrays_[left]->Value(positions_[left]);
-        const int64_t right_sequence = sequence_arrays_[right]->Value(positions_[right]);
-        if (left_sequence != right_sequence) {
-            return left_sequence < right_sequence;
-        }
-        return left < right;
-    }
-
-    struct SourceGreater {
-        RawBatchReader* reader;
-
-        bool operator()(size_t left, size_t right) const {
-            return reader->Less(right, left);
-        }
-    };
-
-    std::vector<StoredBatch> batches_;
-    std::vector<int64_t> positions_;
-    std::vector<int32_t> key_field_indexes_;
-    std::shared_ptr<FieldsComparator> key_comparator_;
-    std::shared_ptr<MemoryPool> memory_pool_;
-    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
-    std::vector<std::shared_ptr<ColumnarBatchContext>> key_contexts_;
-    std::vector<std::shared_ptr<arrow::Int64Array>> sequence_arrays_;
-    std::priority_queue<size_t, std::vector<size_t>, SourceGreater> heap_;
+    std::shared_ptr<arrow::StructArray> data_;
     std::shared_ptr<Metrics> metrics_;
 };
 
@@ -323,13 +179,8 @@ class RawBatchReader final : public BatchReader {
 
 class PrimaryKeyRealtimeStore::Impl {
  public:
-    Impl(std::shared_ptr<arrow::Schema> prepared_schema, std::vector<int32_t> key_field_indexes,
-         const std::shared_ptr<FieldsComparator>& key_comparator,
-         const std::shared_ptr<MemoryPool>& memory_pool)
-        : prepared_schema_(std::move(prepared_schema)),
-          key_field_indexes_(std::move(key_field_indexes)),
-          key_comparator_(key_comparator),
-          memory_pool_(memory_pool) {}
+    explicit Impl(std::shared_ptr<arrow::Schema> prepared_schema)
+        : prepared_schema_(std::move(prepared_schema)) {}
 
     Status Write(RealtimeWriteBatch&& write_batch) {
         if (!write_batch.batch || !write_batch.batch->GetData()) {
@@ -349,7 +200,6 @@ class PrimaryKeyRealtimeStore::Impl {
         }
         std::shared_ptr<arrow::StructArray> prepared =
             checked_pointer_cast<arrow::StructArray>(array);
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(prepared->ValidateFull());
         std::lock_guard<std::mutex> lock(mutex_);
         if (last_offset_ && write_batch.offset_range.begin != last_offset_.value()) {
             return Status::Invalid("PK real-time offset ranges must be contiguous");
@@ -381,9 +231,9 @@ class PrimaryKeyRealtimeStore::Impl {
             return Status::Invalid("segment was not created by the PK real-time store");
         }
         std::vector<std::unique_ptr<BatchReader>> readers;
-        if (!segment->Batches().empty()) {
-            readers.push_back(std::make_unique<RawBatchReader>(
-                segment->Batches(), key_field_indexes_, key_comparator_, memory_pool_));
+        readers.reserve(segment->Batches().size());
+        for (const StoredBatch& batch : segment->Batches()) {
+            readers.push_back(std::make_unique<StoredBatchReader>(batch));
         }
         return readers;
     }
@@ -407,13 +257,10 @@ class PrimaryKeyRealtimeStore::Impl {
             return Status::Invalid("read view was not created by the PK real-time store");
         }
         std::vector<std::unique_ptr<BatchReader>> readers;
-        std::vector<StoredBatch> batches;
         for (const std::shared_ptr<Segment>& segment : typed->Segments()) {
-            batches.insert(batches.end(), segment->Batches().begin(), segment->Batches().end());
-        }
-        if (!batches.empty()) {
-            readers.push_back(std::make_unique<RawBatchReader>(
-                std::move(batches), key_field_indexes_, key_comparator_, memory_pool_));
+            for (const StoredBatch& batch : segment->Batches()) {
+                readers.push_back(std::make_unique<StoredBatchReader>(batch));
+            }
         }
         return readers;
     }
@@ -439,9 +286,6 @@ class PrimaryKeyRealtimeStore::Impl {
 
  private:
     std::shared_ptr<arrow::Schema> prepared_schema_;
-    std::vector<int32_t> key_field_indexes_;
-    std::shared_ptr<FieldsComparator> key_comparator_;
-    std::shared_ptr<MemoryPool> memory_pool_;
     mutable std::mutex mutex_;
     std::vector<StoredBatch> building_;
     std::vector<std::shared_ptr<Segment>> sealed_;
@@ -454,31 +298,10 @@ PrimaryKeyRealtimeStore::PrimaryKeyRealtimeStore(std::unique_ptr<Impl>&& impl)
 PrimaryKeyRealtimeStore::~PrimaryKeyRealtimeStore() = default;
 
 Result<std::shared_ptr<PrimaryKeyRealtimeStore>> PrimaryKeyRealtimeStore::Create(
-    const std::shared_ptr<arrow::Schema>& prepared_schema,
-    const std::vector<std::string>& trimmed_primary_keys,
-    const std::shared_ptr<MemoryPool>& memory_pool) {
+    const std::shared_ptr<arrow::Schema>& prepared_schema) {
     PAIMON_RETURN_NOT_OK(ValidatePreparedTransportSchema(prepared_schema));
-    if (trimmed_primary_keys.empty() || !memory_pool) {
-        return Status::Invalid("PK primary keys are empty or memory pool is null");
-    }
-    std::vector<int32_t> key_field_indexes;
-    std::vector<DataField> key_fields;
-    key_field_indexes.reserve(trimmed_primary_keys.size());
-    key_fields.reserve(trimmed_primary_keys.size());
-    for (const std::string& key : trimmed_primary_keys) {
-        const int32_t field_index = prepared_schema->GetFieldIndex(key);
-        if (field_index < 3) {
-            return Status::Invalid("PK field is missing from prepared schema: ", key);
-        }
-        key_field_indexes.push_back(field_index);
-        PAIMON_ASSIGN_OR_RAISE(DataField field, DataField::ConvertArrowFieldToDataField(
-                                                    prepared_schema->field(field_index)));
-        key_fields.push_back(std::move(field));
-    }
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FieldsComparator> key_comparator,
-                           FieldsComparator::Create(key_fields, /*is_ascending_order=*/true));
-    return std::shared_ptr<PrimaryKeyRealtimeStore>(new PrimaryKeyRealtimeStore(
-        std::make_unique<Impl>(prepared_schema, key_field_indexes, key_comparator, memory_pool)));
+    return std::shared_ptr<PrimaryKeyRealtimeStore>(
+        new PrimaryKeyRealtimeStore(std::make_unique<Impl>(prepared_schema)));
 }
 Status PrimaryKeyRealtimeStore::Write(RealtimeWriteBatch&& batch) {
     return impl_->Write(std::move(batch));
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.h b/src/paimon/core/realtime/primary_key_realtime_store.h
index 52f9a6076..35f04485b 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.h
+++ b/src/paimon/core/realtime/primary_key_realtime_store.h
@@ -30,7 +30,6 @@ class Schema;
 namespace paimon {
 
 class CoreOptions;
-class MemoryPool;
 class TableSchema;
 
 Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options, const TableSchema& schema);
@@ -39,9 +38,7 @@ Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options, const Table
 class PrimaryKeyRealtimeStore final : public RealtimeStore {
  public:
     static Result<std::shared_ptr<PrimaryKeyRealtimeStore>> Create(
-        const std::shared_ptr<arrow::Schema>& prepared_schema,
-        const std::vector<std::string>& trimmed_primary_keys,
-        const std::shared_ptr<MemoryPool>& memory_pool);
+        const std::shared_ptr<arrow::Schema>& prepared_schema);
 
     ~PrimaryKeyRealtimeStore() override;
 
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index 384b937cb..49da66fc2 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -18,9 +18,7 @@
 
 #include "paimon/core/realtime/primary_key_realtime_store.h"
 
-#include <algorithm>
 #include <memory>
-#include <numeric>
 #include <optional>
 #include <string>
 #include <utility>
@@ -29,7 +27,6 @@
 #include "arrow/api.h"
 #include "arrow/c/bridge.h"
 #include "arrow/ipc/json_simple.h"
-#include "fmt/format.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/arrow/status_utils.h"
@@ -168,9 +165,8 @@ TEST(PrimaryKeyRealtimeStoreOptionsTest, TestRejectsEnabledGlobalIndex) {
 }
 
 TEST(PrimaryKeyRealtimeStoreTest, TestWriteAndSealValidation) {
-    ASSERT_OK_AND_ASSIGN(
-        std::shared_ptr<PrimaryKeyRealtimeStore> store,
-        PrimaryKeyRealtimeStore::Create(PreparedSchema(), {"id"}, GetDefaultPool()));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                         PrimaryKeyRealtimeStore::Create(PreparedSchema()));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
                          store->SealForCommit());
     ASSERT_FALSE(segment.has_value());
@@ -221,16 +217,14 @@ TEST(PrimaryKeyRealtimeStoreTest, TestBadTransportPrefix) {
     invalid_fields.push_back(std::move(wrong_offset_id));
 
     for (const arrow::FieldVector& fields : invalid_fields) {
-        ASSERT_NOK_WITH_MSG(
-            PrimaryKeyRealtimeStore::Create(arrow::schema(fields), {"id"}, GetDefaultPool()),
-            "prepared schema field");
+        ASSERT_NOK_WITH_MSG(PrimaryKeyRealtimeStore::Create(arrow::schema(fields)),
+                            "prepared schema field");
     }
 }
 
-TEST(PrimaryKeyRealtimeStoreTest, TestCommitBatches) {
-    ASSERT_OK_AND_ASSIGN(
-        std::shared_ptr<PrimaryKeyRealtimeStore> store,
-        PrimaryKeyRealtimeStore::Create(PreparedSchema(), {"id"}, GetDefaultPool()));
+TEST(PrimaryKeyRealtimeStoreTest, TestCommitReaderPerStoredBatch) {
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                         PrimaryKeyRealtimeStore::Create(PreparedSchema()));
     ASSERT_OK(store->Write(RealtimeWriteBatch{
         MakeBatch(R"([[1, 6, 1, 1, "before"], [0, 5, 0, 3, "three"]])"), OffsetRange(0, 2)}));
     ASSERT_OK(store->Write(
@@ -240,21 +234,23 @@ TEST(PrimaryKeyRealtimeStoreTest, TestCommitBatches) {
     ASSERT_TRUE(segment.has_value());
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
                          store->CreateCommitReaders(segment.value()));
-    ASSERT_EQ(1, readers.size());
+    ASSERT_EQ(2, readers.size());
     ASSERT_OK_AND_ASSIGN(std::string actual, ReadJson(readers));
     ASSERT_EQ(
-        "-- is_valid: all not null\n-- child 0 type: int8\n  [\n    1,\n    2,\n    0\n  ]\n-- "
-        "child 1 type: int64\n  [\n    6,\n    7,\n    5\n  ]\n-- child 2 type: int64\n  [\n    "
-        "1,\n    2,\n    0\n  ]\n-- child 3 type: int64\n  [\n    1,\n    2,\n    3\n  ]\n-- child "
-        "4 type: string\n  [\n    \"before\",\n    \"after\",\n    \"three\"\n  ]",
+        "-- is_valid: all not null\n-- child 0 type: int8\n  [\n    1,\n    0,\n    2\n  ]\n-- "
+        "child 1 type: int64\n  [\n    6,\n    5,\n    7\n  ]\n-- child 2 type: int64\n  [\n    "
+        "1,\n    0,\n    2\n  ]\n-- child 3 type: int64\n  [\n    1,\n    3,\n    2\n  ]\n-- child "
+        "4 type: string\n  [\n    \"before\",\n    \"three\",\n    \"after\"\n  ]",
         actual);
-    readers[0]->Close();
+    for (const std::unique_ptr<BatchReader>& reader : readers) {
+        reader->Close();
+    }
 }
 
 TEST(PrimaryKeyRealtimeStoreTest, TestCommitReaderExportsZeroOffsets) {
     std::shared_ptr<arrow::Schema> schema = NestedPreparedSchema();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(schema, {"id"}, GetDefaultPool()));
+                         PrimaryKeyRealtimeStore::Create(schema));
     ASSERT_OK(store->Write(RealtimeWriteBatch{
         MakeBatch(schema, R"([[0, 1, 0, 1, ["one", [1, 2]]], [0, 2, 1, 2, ["two", [3, 4]]]])"),
         OffsetRange(0, 2)}));
@@ -273,77 +269,9 @@ TEST(PrimaryKeyRealtimeStoreTest, TestCommitReaderExportsZeroOffsets) {
     ASSERT_TRUE(BatchReader::IsEofBatch(batch));
 }
 
-TEST(PrimaryKeyRealtimeStoreTest, TestHeapMergeAcrossBatches) {
-    constexpr int64_t kSourceCount = 2057;
-    constexpr int64_t kKeyCount = 257;
-    ASSERT_OK_AND_ASSIGN(
-        std::shared_ptr<PrimaryKeyRealtimeStore> store,
-        PrimaryKeyRealtimeStore::Create(PreparedSchema(), {"id"}, GetDefaultPool()));
-    for (int64_t source = 0; source < kSourceCount; ++source) {
-        const int64_t id = (source * 149) % kKeyCount;
-        const std::string json =
-            fmt::format(R"([[0, {}, {}, {}, "v{}"]])", source, source, id, source);
-        ASSERT_OK(
-            store->Write(RealtimeWriteBatch{MakeBatch(json), OffsetRange(source, source + 1)}));
-    }
-    ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
-                         store->SealForCommit());
-    ASSERT_TRUE(segment.has_value());
-    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
-                         store->CreateCommitReaders(segment.value()));
-    ASSERT_EQ(1, readers.size());
-
-    std::vector<int64_t> expected_sources(kSourceCount);
-    std::iota(expected_sources.begin(), expected_sources.end(), 0);
-    std::sort(expected_sources.begin(), expected_sources.end(), [=](int64_t left, int64_t right) {
-        const int64_t left_id = (left * 149) % kKeyCount;
-        const int64_t right_id = (right * 149) % kKeyCount;
-        return left_id != right_id ? left_id < right_id : left < right;
-    });
-
-    int64_t output_row = 0;
-    int64_t output_batches = 0;
-    while (true) {
-        ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, readers[0]->NextBatch());
-        if (BatchReader::IsEofBatch(batch)) {
-            break;
-        }
-        ASSERT_LE(batch.first->length, 1024);
-        ASSERT_GT(batch.first->length, 0);
-        ++output_batches;
-        arrow::Result<std::shared_ptr<arrow::Array>> imported_result =
-            arrow::ImportArray(batch.first.get(), batch.second.get());
-        ASSERT_TRUE(imported_result.ok()) << imported_result.status().ToString();
-        std::shared_ptr<arrow::Array> imported = std::move(imported_result).ValueOrDie();
-        std::shared_ptr<arrow::StructArray> array =
-            std::dynamic_pointer_cast<arrow::StructArray>(imported);
-        ASSERT_NE(nullptr, array);
-        ASSERT_EQ(PreparedSchema()->ToString(), arrow::schema(array->type()->fields())->ToString());
-        std::shared_ptr<arrow::Int64Array> sequences =
-            std::dynamic_pointer_cast<arrow::Int64Array>(array->field(1));
-        std::shared_ptr<arrow::Int64Array> ids =
-            std::dynamic_pointer_cast<arrow::Int64Array>(array->field(3));
-        std::shared_ptr<arrow::StringArray> values =
-            std::dynamic_pointer_cast<arrow::StringArray>(array->field(4));
-        ASSERT_NE(nullptr, sequences);
-        ASSERT_NE(nullptr, ids);
-        ASSERT_NE(nullptr, values);
-        for (int64_t row = 0; row < array->length(); ++row, ++output_row) {
-            ASSERT_LT(output_row, kSourceCount);
-            const int64_t source = expected_sources[output_row];
-            ASSERT_EQ(source, sequences->Value(row));
-            ASSERT_EQ((source * 149) % kKeyCount, ids->Value(row));
-            ASSERT_EQ(fmt::format("v{}", source), values->GetString(row));
-        }
-    }
-    ASSERT_EQ(kSourceCount, output_row);
-    ASSERT_EQ(3, output_batches);
-}
-
-TEST(PrimaryKeyRealtimeStoreTest, TestCloseUnreadMultiSourceReader) {
-    ASSERT_OK_AND_ASSIGN(
-        std::shared_ptr<PrimaryKeyRealtimeStore> store,
-        PrimaryKeyRealtimeStore::Create(PreparedSchema(), {"id"}, GetDefaultPool()));
+TEST(PrimaryKeyRealtimeStoreTest, TestCloseUnreadBatchReaders) {
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                         PrimaryKeyRealtimeStore::Create(PreparedSchema()));
     ASSERT_OK(
         store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 10, 0, 1, "a"]])"), OffsetRange(0, 1)}));
     ASSERT_OK(
@@ -355,15 +283,15 @@ TEST(PrimaryKeyRealtimeStoreTest, TestCloseUnreadMultiSourceReader) {
     ASSERT_TRUE(segment.has_value());
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
                          store->CreateCommitReaders(segment.value()));
-    ASSERT_EQ(1, readers.size());
-
-    readers[0]->Close();
+    ASSERT_EQ(3, readers.size());
+    for (const std::unique_ptr<BatchReader>& reader : readers) {
+        reader->Close();
+    }
 }
 
 TEST(PrimaryKeyRealtimeStoreTest, TestReclaimKeepsReadView) {
-    ASSERT_OK_AND_ASSIGN(
-        std::shared_ptr<PrimaryKeyRealtimeStore> store,
-        PrimaryKeyRealtimeStore::Create(PreparedSchema(), {"id"}, GetDefaultPool()));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                         PrimaryKeyRealtimeStore::Create(PreparedSchema()));
     ASSERT_OK(
         store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 0, 4, 1, "one"]])"), OffsetRange(4, 5)}));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
@@ -373,10 +301,9 @@ TEST(PrimaryKeyRealtimeStoreTest, TestReclaimKeepsReadView) {
     ASSERT_EQ(std::optional<OffsetRange>(OffsetRange(4, 5)), view->GetOffsetRange());
 }
 
-TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderCardinalityIsConstant) {
-    ASSERT_OK_AND_ASSIGN(
-        std::shared_ptr<PrimaryKeyRealtimeStore> store,
-        PrimaryKeyRealtimeStore::Create(PreparedSchema(), {"id"}, GetDefaultPool()));
+TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderPerStoredBatch) {
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                         PrimaryKeyRealtimeStore::Create(PreparedSchema()));
     ASSERT_OK(
         store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 1, 0, 2, "two"]])"), OffsetRange(0, 1)}));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
@@ -389,7 +316,7 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderCardinalityIsConstant) {
                                  /*enable_predicate_pushdown=*/false};
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
                          store->CreateQueryReaders(view, /*offset_begin=*/0, context));
-    ASSERT_EQ(1, readers.size());
+    ASSERT_EQ(2, readers.size());
     ASSERT_OK_AND_ASSIGN(std::string actual, ReadJson(readers));
     ASSERT_NE(std::string::npos, actual.find("\"one\""));
     ASSERT_NE(std::string::npos, actual.find("\"two\""));
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index c9f719b7e..0ea4c61d6 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -49,14 +49,7 @@ namespace paimon {
 namespace {
 
 bool SameMode(const RealtimeStoreCreateConfig& left, const RealtimeStoreCreateConfig& right) {
-    if (left.index() != right.index()) {
-        return false;
-    }
-    if (const auto* left_pk = std::get_if<PrimaryKeyRealtimeStoreCreateConfig>(&left)) {
-        const auto& right_pk = std::get<PrimaryKeyRealtimeStoreCreateConfig>(right);
-        return left_pk->trimmed_primary_keys == right_pk.trimmed_primary_keys;
-    }
-    return true;
+    return left.index() == right.index();
 }
 
 std::string PartitionToString(const std::map<std::string, std::string>& partition) {
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index f10e93858..173981527 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -2349,7 +2349,7 @@ TEST_F(RealtimeWriteInteTest, TestPkCompaction) {
               final_rows);
 }
 
-TEST_F(RealtimeWriteInteTest, TestPkPluginContract) {
+TEST_F(RealtimeWriteInteTest, TestPkMultipleStoredBatchesMergeForQueryAndCommit) {
     CreatePkTable();
     auto factory = MakeDecoratingFactory<SplitCommitReaderRealtimeStore>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
@@ -2357,23 +2357,28 @@ TEST_F(RealtimeWriteInteTest, TestPkPluginContract) {
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                          CreateRealtimeWriter(realtime_context));
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
-                         MakeBatch({Row{4, "four", "p0"}, Row{3, "three", "p0"}},
+                         MakeBatch({Row{4, "four", "p0"}, Row{2, "two", "p0"}, Row{1, "one", "p0"}},
                                    /*partitioned=*/false));
     ASSERT_OK(writer->Write(std::move(first_batch)));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
-                         MakeBatch({Row{2, "two", "p0"}, Row{1, "one", "p0"}},
-                                   /*partitioned=*/false));
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<RecordBatch> second_batch,
+        MakeBatch({Row{3, "three", "p0"}, Row{2, "deleted", "p0"}, Row{1, "one-new", "p0"}},
+                  /*partitioned=*/false, /*bucket=*/0,
+                  {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::DELETE,
+                   RecordBatch::RowKind::UPDATE_AFTER}));
     ASSERT_OK(writer->Write(std::move(second_batch)));
 
+    const std::vector<Row> expected = {{1, "one-new", "p0"}, {3, "three", "p0"}, {4, "four", "p0"}};
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> query_rows, ReadRows(realtime_context));
+    ASSERT_EQ(expected, query_rows);
+
     ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                          writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
     ASSERT_EQ(1, progress.size());
-    ASSERT_EQ(OffsetRange(0, 4), progress[0].offset_range);
+    ASSERT_EQ(OffsetRange(0, 6), progress[0].offset_range);
     ASSERT_OK(Commit(progress, /*commit_identifier=*/0));
     ASSERT_OK_AND_ASSIGN(std::vector<Row> rows, ReadRows());
-    ASSERT_EQ((std::vector<Row>{
-                  {1, "one", "p0"}, {2, "two", "p0"}, {3, "three", "p0"}, {4, "four", "p0"}}),
-              rows);
+    ASSERT_EQ(expected, rows);
     ASSERT_OK(writer->Close());
 }
 
@@ -2445,7 +2450,7 @@ TEST_F(RealtimeWriteInteTest, TestPkQueryReaderCloseFailure) {
         state->query_null_index = null_index;
         ASSERT_NOK_WITH_MSG(CreateQueryReader(realtime_context),
                             "PK real-time store returned a null query reader");
-        ASSERT_EQ(null_index + 1, state->query_close_count->load(std::memory_order_acquire));
+        ASSERT_EQ((null_index + 1) * 2, state->query_close_count->load(std::memory_order_acquire));
     }
     ASSERT_OK(writer->Close());
 }

From 808cc593d2fab91caccd715ab7f6122c30a4fe49 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Tue, 25 Aug 2026 15:56:43 +0800
Subject: [PATCH 31/62] refactor(realtime): simplify PK offset coverage

---
 .../realtime/prepared_key_value_reader.cpp    | 53 +++++++------------
 test/inte/realtime_write_inte_test.cpp        | 23 +++++---
 2 files changed, 33 insertions(+), 43 deletions(-)

diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index c17513aef..ae575ca58 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -18,9 +18,10 @@
 
 #include "paimon/core/realtime/prepared_key_value_reader.h"
 
+#include <algorithm>
 #include <cstdint>
+#include <limits>
 #include <memory>
-#include <mutex>
 #include <optional>
 #include <unordered_map>
 #include <utility>
@@ -29,11 +30,8 @@
 #include "arrow/array/array_base.h"
 #include "arrow/array/array_nested.h"
 #include "arrow/array/array_primitive.h"
-#include "arrow/array/builder_primitive.h"
-#include "arrow/buffer.h"
 #include "arrow/c/bridge.h"
 #include "arrow/type.h"
-#include "arrow/util/bit_util.h"
 #include "fmt/format.h"
 #include "paimon/common/data/columnar/columnar_batch_context.h"
 #include "paimon/common/data/columnar/columnar_row_ref.h"
@@ -75,42 +73,35 @@ Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
 
 class RealtimeOffsetCoverage {
  public:
-    static Result<std::shared_ptr<RealtimeOffsetCoverage>> Create(
-        const OffsetRange& sealed_offsets, size_t reader_count,
-        const std::shared_ptr<arrow::MemoryPool>& arrow_pool) {
+    static Result<std::shared_ptr<RealtimeOffsetCoverage>> Create(const OffsetRange& sealed_offsets,
+                                                                  size_t reader_count) {
         if (sealed_offsets.begin < 0 || sealed_offsets.end < sealed_offsets.begin) {
             return Status::Invalid("PK real-time store returned an invalid sealed offset range");
         }
-        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-            std::shared_ptr<arrow::Buffer> seen_offsets,
-            arrow::AllocateEmptyBitmap(sealed_offsets.Count(), arrow_pool.get()));
-        return std::shared_ptr<RealtimeOffsetCoverage>(new RealtimeOffsetCoverage(
-            sealed_offsets, reader_count, std::move(seen_offsets), arrow_pool));
+        return std::shared_ptr<RealtimeOffsetCoverage>(
+            new RealtimeOffsetCoverage(sealed_offsets, reader_count));
     }
 
     Status Add(const arrow::Int64Array& offsets) {
-        std::lock_guard<std::mutex> lock(mutex_);
         for (int64_t row = 0; row < offsets.length(); ++row) {
             const int64_t offset = offsets.Value(row);
             if (offset < sealed_offsets_.begin || offset >= sealed_offsets_.end) {
                 return Status::Invalid(
                     "PK real-time store commit reader offset is outside the sealed range");
             }
-            const int64_t index = offset - sealed_offsets_.begin;
-            if (arrow::bit_util::GetBit(seen_offsets_->data(), index)) {
-                return Status::Invalid(
-                    "PK real-time store commit readers contain duplicate REALTIME_OFFSET");
-            }
-            arrow::bit_util::SetBit(seen_offsets_->mutable_data(), index);
+            min_seen_offset_ = std::min(min_seen_offset_, offset);
+            max_seen_offset_ = std::max(max_seen_offset_, offset);
             ++seen_count_;
         }
         return Status::OK();
     }
 
     Status FinishReader() {
-        std::lock_guard<std::mutex> lock(mutex_);
         ++finished_reader_count_;
-        if (finished_reader_count_ == reader_count_ && seen_count_ != sealed_offsets_.Count()) {
+        if (finished_reader_count_ == reader_count_ &&
+            (seen_count_ != sealed_offsets_.Count() ||
+             (seen_count_ > 0 && (min_seen_offset_ != sealed_offsets_.begin ||
+                                  max_seen_offset_ != sealed_offsets_.end - 1)))) {
             return Status::Invalid(
                 "PK real-time store commit readers did not cover the sealed range");
         }
@@ -118,21 +109,15 @@ class RealtimeOffsetCoverage {
     }
 
  private:
-    RealtimeOffsetCoverage(const OffsetRange& sealed_offsets, size_t reader_count,
-                           std::shared_ptr<arrow::Buffer> seen_offsets,
-                           const std::shared_ptr<arrow::MemoryPool>& arrow_pool)
-        : sealed_offsets_(sealed_offsets),
-          reader_count_(reader_count),
-          arrow_pool_(arrow_pool),
-          seen_offsets_(std::move(seen_offsets)) {}
+    RealtimeOffsetCoverage(const OffsetRange& sealed_offsets, size_t reader_count)
+        : sealed_offsets_(sealed_offsets), reader_count_(reader_count) {}
 
     OffsetRange sealed_offsets_;
     size_t reader_count_;
-    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
-    std::shared_ptr<arrow::Buffer> seen_offsets_;
+    int64_t min_seen_offset_ = std::numeric_limits<int64_t>::max();
+    int64_t max_seen_offset_ = std::numeric_limits<int64_t>::min();
     int64_t seen_count_ = 0;
     size_t finished_reader_count_ = 0;
-    std::mutex mutex_;
 };
 
 Status CheckPreparedField(const std::shared_ptr<arrow::Schema>& schema, int32_t field_idx,
@@ -736,10 +721,8 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> AdaptPreparedCommitBa
             return Status::Invalid("PK real-time store returned a null commit reader");
         }
     }
-    std::shared_ptr<arrow::MemoryPool> arrow_pool = GetArrowPool(memory_pool);
-    PAIMON_ASSIGN_OR_RAISE(
-        std::shared_ptr<RealtimeOffsetCoverage> offset_coverage,
-        RealtimeOffsetCoverage::Create(sealed_offsets, readers.size(), arrow_pool));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeOffsetCoverage> offset_coverage,
+                           RealtimeOffsetCoverage::Create(sealed_offsets, readers.size()));
     adapted_readers.reserve(readers.size());
     for (std::unique_ptr<BatchReader>& reader : readers) {
         PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> adapted_reader,
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index 173981527..ed75b554e 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -538,7 +538,7 @@ class FailAfterPhysicalFileRealtimeStore final : public DelegatingRealtimeStore
     std::shared_ptr<std::atomic<bool>> saw_artifacts_;
 };
 
-enum class CommitReaderMalformation { DROP_LAST, UNSORTED, SUBSTITUTE_OFFSET };
+enum class CommitReaderMalformation { DROP_LAST, UNSORTED, DUPLICATE_OFFSET, OUT_OF_RANGE_OFFSET };
 
 class CorruptingBatchReader final : public BatchReader {
  public:
@@ -552,8 +552,10 @@ class CorruptingBatchReader final : public BatchReader {
                 return DropLast();
             case CommitReaderMalformation::UNSORTED:
                 return SwapFirstTwo();
-            case CommitReaderMalformation::SUBSTITUTE_OFFSET:
-                return SubstituteOffset();
+            case CommitReaderMalformation::DUPLICATE_OFFSET:
+                return SubstituteOffset(/*offset=*/0);
+            case CommitReaderMalformation::OUT_OF_RANGE_OFFSET:
+                return SubstituteOffset(/*offset=*/-1);
         }
         return Status::Invalid("unknown commit reader malformation");
     }
@@ -613,7 +615,7 @@ class CorruptingBatchReader final : public BatchReader {
         return ReadBatch(std::move(output), std::move(schema));
     }
 
-    Result<ReadBatch> SubstituteOffset() {
+    Result<ReadBatch> SubstituteOffset(int64_t offset) {
         PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, delegate_->NextBatch());
         if (BatchReader::IsEofBatch(batch)) {
             return batch;
@@ -634,7 +636,7 @@ class CorruptingBatchReader final : public BatchReader {
         arrow::Int64Builder builder;
         PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Reserve(offsets->length()));
         for (int64_t row = 0; row < offsets->length(); ++row) {
-            builder.UnsafeAppend(0);
+            builder.UnsafeAppend(offset);
         }
         std::shared_ptr<arrow::Array> substituted_offsets;
         PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&substituted_offsets));
@@ -2387,9 +2389,14 @@ TEST_F(RealtimeWriteInteTest, TestPkRejectsMalformedCoverage) {
                                            "commit readers did not cover the sealed range");
 }
 
-TEST_F(RealtimeWriteInteTest, TestPkRejectsEqualCardinalityOffsetSubstitution) {
-    CheckPkRejectsCommitReaderMalformation(CommitReaderMalformation::SUBSTITUTE_OFFSET,
-                                           "duplicate REALTIME_OFFSET");
+TEST_F(RealtimeWriteInteTest, TestPkRejectsDuplicateOffset) {
+    CheckPkRejectsCommitReaderMalformation(CommitReaderMalformation::DUPLICATE_OFFSET,
+                                           "commit readers did not cover the sealed range");
+}
+
+TEST_F(RealtimeWriteInteTest, TestPkRejectsOutOfRangeOffset) {
+    CheckPkRejectsCommitReaderMalformation(CommitReaderMalformation::OUT_OF_RANGE_OFFSET,
+                                           "offset is outside the sealed range");
 }
 
 TEST_F(RealtimeWriteInteTest, TestPkRejectsUnsortedPluginRows) {

From 7e592537123f6547ad7eaaecdd456660e1e5a5cc Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Tue, 25 Aug 2026 17:58:31 +0800
Subject: [PATCH 32/62] refactor(realtime): simplify stores around
 framework-owned PK offsets

---
 include/paimon/realtime/realtime_store.h      | 17 +++----
 .../merged_key_value_record_reader_test.cpp   | 34 +------------
 .../operation/key_value_file_store_write.cpp  |  6 +--
 .../realtime/arrow_realtime_store_factory.cpp | 22 ++++----
 .../realtime/arrow_realtime_store_test.cpp    | 15 +++++-
 .../realtime/prepared_key_value_reader.cpp    | 50 ++-----------------
 .../core/realtime/prepared_key_value_reader.h |  5 +-
 .../realtime/primary_key_realtime_store.cpp   |  5 --
 .../primary_key_realtime_store_test.cpp       |  6 ---
 .../realtime/realtime_append_only_writer.cpp  |  4 +-
 .../core/realtime/realtime_context_impl.cpp   | 10 ++--
 .../core/realtime/realtime_context_impl.h     |  2 +-
 .../core/realtime/realtime_context_test.cpp   | 27 ++++++++--
 .../realtime/realtime_primary_key_writer.cpp  |  2 +-
 .../table/source/key_value_table_read.cpp     | 11 ++--
 test/inte/realtime_write_inte_test.cpp        | 38 +-------------
 16 files changed, 78 insertions(+), 176 deletions(-)

diff --git a/include/paimon/realtime/realtime_store.h b/include/paimon/realtime/realtime_store.h
index e61051e29..241413b31 100644
--- a/include/paimon/realtime/realtime_store.h
+++ b/include/paimon/realtime/realtime_store.h
@@ -25,7 +25,6 @@
 #include <optional>
 #include <string>
 #include <utility>
-#include <variant>
 #include <vector>
 
 #include "arrow/c/abi.h"
@@ -43,15 +42,11 @@ namespace paimon {
 class MemoryPool;
 class Predicate;
 
-struct PAIMON_EXPORT AppendRealtimeStoreCreateConfig {
-    StatisticsMode statistics_mode;
+enum class PAIMON_EXPORT RealtimeStoreMode {
+    APPEND_ONLY,
+    PRIMARY_KEY,
 };
 
-struct PAIMON_EXPORT PrimaryKeyRealtimeStoreCreateConfig {};
-
-using RealtimeStoreCreateConfig =
-    std::variant<AppendRealtimeStoreCreateConfig, PrimaryKeyRealtimeStoreCreateConfig>;
-
 /// Parameters used by a `RealtimeStoreFactory` to create a store.
 struct PAIMON_EXPORT RealtimeStoreCreateRequest {
     /// Schema whose ownership is transferred to the factory. Append mode receives the complete
@@ -66,8 +61,10 @@ struct PAIMON_EXPORT RealtimeStoreCreateRequest {
     std::map<std::string, std::string> partition;
     /// Bucket identifying the store within its partition.
     int32_t bucket = -1;
-    /// Mode-specific store configuration.
-    RealtimeStoreCreateConfig mode_config;
+    /// Table mode implemented by the store.
+    RealtimeStoreMode mode = RealtimeStoreMode::APPEND_ONLY;
+    /// Statistics collected by append-only stores.
+    StatisticsMode statistics_mode = StatisticsMode::NONE;
 };
 
 /// A record batch and its framework-assigned contiguous offset range.
diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index 81a1f133e..3ba81f03f 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -71,15 +71,8 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderForTest(
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
     const std::shared_ptr<MemoryPool>& memory_pool) {
-    if (!key_schema) {
-        return Status::Invalid("prepared key schema cannot be null");
-    }
-    PAIMON_ASSIGN_OR_RAISE(std::vector<DataField> key_fields,
-                           DataField::ConvertArrowSchemaToDataFields(key_schema));
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FieldsComparator> key_comparator,
-                           FieldsComparator::Create(key_fields, /*is_ascending_order=*/true));
     return AdaptPreparedBatchReader(std::move(reader), prepared_schema, visible_offsets, key_schema,
-                                    value_schema, key_comparator, memory_pool);
+                                    value_schema, memory_pool);
 }
 
 class TrackingBatchReader : public BatchReader {
@@ -266,31 +259,6 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderRejectsReversedVisibleO
     ASSERT_NOK_WITH_MSG(result, "prepared visible offset range begin exceeds end");
 }
 
-TEST_F(MergedKeyValueRecordReaderTest, TestRejectsUnsortedPluginRowsAcrossBatches) {
-    std::vector<DataField> value_fields = {DataField(0, arrow::field("id", arrow::int32()))};
-    std::shared_ptr<arrow::Schema> value_schema =
-        DataField::ConvertDataFieldsToArrowSchema(value_fields);
-    std::shared_ptr<arrow::Schema> key_schema = arrow::schema({value_schema->field(0)});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema(value_schema->fields());
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
-            [0, 10, 0, 2],
-            [0, 11, 1, 1]
-        ])")
-            .ValueOrDie();
-    auto batch_reader =
-        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, /*batch_size=*/1);
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, std::nullopt,
-                                        key_schema, value_schema, pool_));
-    Result<std::vector<KeyValue>> result =
-        ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
-                                                   KeyValueRecordReader::Iterator>(reader.get());
-    ASSERT_NOK_WITH_MSG(result, "not globally sorted by primary key and sequence number");
-}
-
 TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderCommitSchema) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Field> extra = MakeField("extra", arrow::int32(), 1);
diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp
index 31889ee84..d8e7f5d15 100644
--- a/src/paimon/core/operation/key_value_file_store_write.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write.cpp
@@ -152,9 +152,9 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
             arrow::ExportSchema(*arrow::schema(std::move(prepared_fields)), c_write_schema.get()));
         PAIMON_ASSIGN_OR_RAISE(
             RealtimeStoreState store_state,
-            realtime_context_impl->GetOrCreateRealtimeStore(RealtimeStoreCreateRequest{
-                std::move(c_write_schema), options_.ToMap(), pool_, partition_map, bucket,
-                PrimaryKeyRealtimeStoreCreateConfig{}}));
+            realtime_context_impl->GetOrCreateRealtimeStore(
+                RealtimeStoreCreateRequest{std::move(c_write_schema), options_.ToMap(), pool_,
+                                           partition_map, bucket, RealtimeStoreMode::PRIMARY_KEY}));
         realtime_store_state = std::move(store_state);
         compact_manager = std::make_shared<NoopCompactManager>();
     } else {
diff --git a/src/paimon/core/realtime/arrow_realtime_store_factory.cpp b/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
index d336394ad..d0d4ae704 100644
--- a/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
+++ b/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
@@ -42,17 +42,19 @@ Result<std::shared_ptr<RealtimeStore>> ArrowRealtimeStoreFactory::Create(
     }
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> imported_schema,
                                       arrow::ImportSchema(request.write_schema.get()));
-    if (std::holds_alternative<AppendRealtimeStoreCreateConfig>(request.mode_config)) {
-        const AppendRealtimeStoreCreateConfig& append_config =
-            std::get<AppendRealtimeStoreCreateConfig>(request.mode_config);
-        std::shared_ptr<arrow::MemoryPool> arrow_pool = GetArrowPool(request.memory_pool);
-        return std::make_shared<ArrowRealtimeStore>(imported_schema, append_config.statistics_mode,
-                                                    request.memory_pool, arrow_pool);
+    switch (request.mode) {
+        case RealtimeStoreMode::APPEND_ONLY: {
+            std::shared_ptr<arrow::MemoryPool> arrow_pool = GetArrowPool(request.memory_pool);
+            return std::make_shared<ArrowRealtimeStore>(imported_schema, request.statistics_mode,
+                                                        request.memory_pool, arrow_pool);
+        }
+        case RealtimeStoreMode::PRIMARY_KEY: {
+            PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                                   PrimaryKeyRealtimeStore::Create(imported_schema));
+            return std::shared_ptr<RealtimeStore>(std::move(store));
+        }
     }
-
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                           PrimaryKeyRealtimeStore::Create(imported_schema));
-    return std::shared_ptr<RealtimeStore>(std::move(store));
+    return Status::Invalid("invalid real-time store mode: ", static_cast<int32_t>(request.mode));
 }
 
 }  // namespace paimon
diff --git a/src/paimon/core/realtime/arrow_realtime_store_test.cpp b/src/paimon/core/realtime/arrow_realtime_store_test.cpp
index f186a8161..0a3e52353 100644
--- a/src/paimon/core/realtime/arrow_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/arrow_realtime_store_test.cpp
@@ -237,7 +237,8 @@ TEST_F(ArrowRealtimeStoreTest, TestFullStatisticsPrunesNonMatchingBatch) {
                                        pool_,
                                        /*partition=*/{},
                                        /*bucket=*/0,
-                                       AppendRealtimeStoreCreateConfig{StatisticsMode::FULL}};
+                                       RealtimeStoreMode::APPEND_ONLY,
+                                       StatisticsMode::FULL};
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeStore> realtime_store,
                          factory.Create(std::move(request)));
     std::shared_ptr<ArrowRealtimeStore> store =
@@ -274,6 +275,18 @@ TEST_F(ArrowRealtimeStoreTest, TestFullStatisticsPrunesNonMatchingBatch) {
     ASSERT_EQ(std::vector<int64_t>({0, 1}), ReadIds(unfiltered_batch));
 }
 
+TEST_F(ArrowRealtimeStoreTest, TestFactoryRejectsInvalidMode) {
+    ArrowRealtimeStoreFactory factory;
+    std::unique_ptr<ArrowSchema> write_schema = MakeReadSchema(schema_);
+    RealtimeStoreCreateRequest request{std::move(write_schema),
+                                       /*options=*/{},
+                                       pool_,
+                                       /*partition=*/{},
+                                       /*bucket=*/0,
+                                       static_cast<RealtimeStoreMode>(-1)};
+    ASSERT_NOK_WITH_MSG(factory.Create(std::move(request)), "invalid real-time store mode: -1");
+}
+
 TEST_F(ArrowRealtimeStoreTest, TestMissingStatisticsRetainsNonMatchingBatch) {
     ASSERT_OK(
         store_->Write(RealtimeWriteBatch{MakeBatch(R"([[0, "a"], [1, "b"]])"), OffsetRange(0, 2)}));
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index ae575ca58..a34ccea43 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -42,7 +42,6 @@
 #include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
-#include "paimon/common/utils/fields_comparator.h"
 #include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/utils/nested_projection_utils.h"
 #include "paimon/macros.h"
@@ -377,7 +376,6 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
                            const std::optional<OffsetRange>& visible_offsets,
                            const std::shared_ptr<arrow::Schema>& key_schema,
                            const std::shared_ptr<arrow::Schema>& value_schema,
-                           const std::shared_ptr<FieldsComparator>& key_comparator,
                            const std::shared_ptr<MemoryPool>& pool,
                            const std::shared_ptr<RealtimeOffsetCoverage>& offset_coverage)
         : reader_(std::move(reader)),
@@ -385,7 +383,6 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
           visible_offsets_(visible_offsets),
           key_schema_(key_schema),
           value_schema_(value_schema),
-          key_comparator_(key_comparator),
           pool_(pool),
           arrow_pool_(GetArrowPool(pool)),
           offset_coverage_(offset_coverage) {}
@@ -506,7 +503,6 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
                                                             value_schema_, arrow_pool_.get()));
             key_ctx_ = std::make_shared<ColumnarBatchContext>(key_fields, pool_);
             value_ctx_ = std::make_shared<ColumnarBatchContext>(value_fields, pool_);
-            PAIMON_RETURN_NOT_OK(ValidateOrdering(key_ctx_, sequence_number_array_));
             if (!SelectVisibleRows(*offset_array)) {
                 continue;
             }
@@ -548,32 +544,6 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
         return Status::OK();
     }
 
-    Status ValidateOrdering(
-        const std::shared_ptr<ColumnarBatchContext>& key_context,
-        const std::shared_ptr<arrow::NumericArray<arrow::Int64Type>>& sequences) {
-        if (sequences->length() == 0) {
-            return Status::OK();
-        }
-        for (int64_t row = 0; row < sequences->length(); ++row) {
-            ColumnarRowRef current_key(key_context, row);
-            if (previous_key_context_) {
-                ColumnarRowRef previous_key(previous_key_context_, previous_key_row_);
-                const int32_t key_comparison =
-                    key_comparator_->CompareTo(previous_key, current_key);
-                if (key_comparison > 0 ||
-                    (key_comparison == 0 && previous_sequence_ > sequences->Value(row))) {
-                    return Status::Invalid(
-                        "PK real-time plugin reader is not globally sorted by primary key and "
-                        "sequence number");
-                }
-            }
-            previous_key_context_ = key_context;
-            previous_key_row_ = row;
-            previous_sequence_ = sequences->Value(row);
-        }
-        return Status::OK();
-    }
-
     bool SelectVisibleRows(const arrow::Int64Array& offsets) {
         if (!visible_offsets_.has_value()) {
             return true;
@@ -614,7 +584,6 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
     std::optional<OffsetRange> visible_offsets_;
     std::shared_ptr<arrow::Schema> key_schema_;
     std::shared_ptr<arrow::Schema> value_schema_;
-    std::shared_ptr<FieldsComparator> key_comparator_;
     std::shared_ptr<MemoryPool> pool_;
     std::shared_ptr<arrow::MemoryPool> arrow_pool_;
     std::shared_ptr<RealtimeOffsetCoverage> offset_coverage_;
@@ -624,9 +593,6 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
     std::shared_ptr<arrow::NumericArray<arrow::Int8Type>> row_kind_array_;
     std::shared_ptr<arrow::NumericArray<arrow::Int64Type>> sequence_number_array_;
     std::optional<std::vector<int64_t>> visible_rows_;
-    std::shared_ptr<ColumnarBatchContext> previous_key_context_;
-    int64_t previous_key_row_ = 0;
-    int64_t previous_sequence_ = 0;
 };
 
 }  // namespace
@@ -651,7 +617,6 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderImpl(
     const std::optional<OffsetRange>& visible_offsets,
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
-    const std::shared_ptr<FieldsComparator>& key_comparator,
     const std::shared_ptr<MemoryPool>& memory_pool,
     const std::shared_ptr<RealtimeOffsetCoverage>& offset_coverage) {
     std::unique_ptr<BatchReader> owned_reader = std::move(reader);
@@ -669,9 +634,6 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderImpl(
     if (!value_schema) {
         return Status::Invalid("prepared value schema cannot be null");
     }
-    if (!key_comparator) {
-        return Status::Invalid("prepared key comparator cannot be null");
-    }
     if (!memory_pool) {
         return Status::Invalid("prepared reader memory pool cannot be null");
     }
@@ -680,9 +642,9 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderImpl(
     if (!visible_offsets.has_value()) {
         PAIMON_RETURN_NOT_OK(ValidateExactCommitSchema(prepared_schema, value_schema));
     }
-    std::unique_ptr<KeyValueRecordReader> result(new PreparedKeyValueReader(
-        std::move(owned_reader), prepared_schema, visible_offsets, key_schema, value_schema,
-        key_comparator, memory_pool, offset_coverage));
+    std::unique_ptr<KeyValueRecordReader> result(
+        new PreparedKeyValueReader(std::move(owned_reader), prepared_schema, visible_offsets,
+                                   key_schema, value_schema, memory_pool, offset_coverage));
     close_guard.Release();
     return result;
 }
@@ -694,10 +656,9 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
     const std::optional<OffsetRange>& visible_offsets,
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
-    const std::shared_ptr<FieldsComparator>& key_comparator,
     const std::shared_ptr<MemoryPool>& memory_pool) {
     return AdaptPreparedBatchReaderImpl(std::move(reader), prepared_schema, visible_offsets,
-                                        key_schema, value_schema, key_comparator, memory_pool,
+                                        key_schema, value_schema, memory_pool,
                                         /*offset_coverage=*/nullptr);
 }
 
@@ -706,7 +667,6 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> AdaptPreparedCommitBa
     const std::shared_ptr<arrow::Schema>& prepared_schema, const OffsetRange& sealed_offsets,
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
-    const std::shared_ptr<FieldsComparator>& key_comparator,
     const std::shared_ptr<MemoryPool>& memory_pool) {
     std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers;
     ScopeGuard readers_guard([&readers, &adapted_readers]() {
@@ -728,7 +688,7 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> AdaptPreparedCommitBa
         PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> adapted_reader,
                                AdaptPreparedBatchReaderImpl(
                                    std::move(reader), prepared_schema, std::nullopt, key_schema,
-                                   value_schema, key_comparator, memory_pool, offset_coverage));
+                                   value_schema, memory_pool, offset_coverage));
         adapted_readers.push_back(std::move(adapted_reader));
     }
     readers_guard.Release();
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.h b/src/paimon/core/realtime/prepared_key_value_reader.h
index 81ae0abc7..4ef4887e6 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.h
+++ b/src/paimon/core/realtime/prepared_key_value_reader.h
@@ -30,7 +30,6 @@
 
 namespace paimon {
 class BatchReader;
-class FieldsComparator;
 class MemoryPool;
 
 /// Validates the required leading fields of a prepared real-time transport schema.
@@ -42,16 +41,14 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
     const std::optional<OffsetRange>& visible_offsets,
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
-    const std::shared_ptr<FieldsComparator>& key_comparator,
     const std::shared_ptr<MemoryPool>& memory_pool);
 
-/// Adapts commit readers and validates that they collectively cover `sealed_offsets` exactly.
+/// Adapts commit readers and validates their offsets against `sealed_offsets`.
 Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> AdaptPreparedCommitBatchReaders(
     std::vector<std::unique_ptr<BatchReader>>&& readers,
     const std::shared_ptr<arrow::Schema>& prepared_schema, const OffsetRange& sealed_offsets,
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
-    const std::shared_ptr<FieldsComparator>& key_comparator,
     const std::shared_ptr<MemoryPool>& memory_pool);
 
 }  // namespace paimon
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index 68f4bf500..7d188609d 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -201,13 +201,9 @@ class PrimaryKeyRealtimeStore::Impl {
         std::shared_ptr<arrow::StructArray> prepared =
             checked_pointer_cast<arrow::StructArray>(array);
         std::lock_guard<std::mutex> lock(mutex_);
-        if (last_offset_ && write_batch.offset_range.begin != last_offset_.value()) {
-            return Status::Invalid("PK real-time offset ranges must be contiguous");
-        }
         building_.push_back(
             StoredBatch{prepared, write_batch.offset_range, GetArrayMemoryUsage(prepared->data())});
         building_memory_usage_ += building_.back().memory_usage;
-        last_offset_ = write_batch.offset_range.end;
         return Status::OK();
     }
 
@@ -290,7 +286,6 @@ class PrimaryKeyRealtimeStore::Impl {
     std::vector<StoredBatch> building_;
     std::vector<std::shared_ptr<Segment>> sealed_;
     uint64_t building_memory_usage_ = 0;
-    std::optional<int64_t> last_offset_;
 };
 
 PrimaryKeyRealtimeStore::PrimaryKeyRealtimeStore(std::unique_ptr<Impl>&& impl)
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index 49da66fc2..6ccceba81 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -178,9 +178,6 @@ TEST(PrimaryKeyRealtimeStoreTest, TestWriteAndSealValidation) {
 
     ASSERT_OK(store->Write(RealtimeWriteBatch{
         MakeBatch(R"([[0, 1, 0, 1, "one"], [0, 2, 1, 2, "two"]])"), OffsetRange(0, 2)}));
-    ASSERT_NOK_WITH_MSG(store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 3, 3, 3, "three"]])"),
-                                                        OffsetRange(3, 4)}),
-                        "offset ranges must be contiguous");
     ASSERT_OK(store->Write(
         RealtimeWriteBatch{MakeBatch(R"([[0, 3, 2, 3, "three"]])"), OffsetRange(2, 3)}));
 
@@ -188,9 +185,6 @@ TEST(PrimaryKeyRealtimeStoreTest, TestWriteAndSealValidation) {
     ASSERT_TRUE(segment.has_value());
     ASSERT_EQ(OffsetRange(0, 3), segment.value()->GetOffsetRange());
     ASSERT_GT(store->GetMemoryUsage(), 0);
-    ASSERT_NOK_WITH_MSG(
-        store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 4, 4, 4, "four"]])"), OffsetRange(4, 5)}),
-        "offset ranges must be contiguous");
     ASSERT_OK(store->Write(
         RealtimeWriteBatch{MakeBatch(R"([[0, 4, 3, 4, "four"]])"), OffsetRange(3, 4)}));
 }
diff --git a/src/paimon/core/realtime/realtime_append_only_writer.cpp b/src/paimon/core/realtime/realtime_append_only_writer.cpp
index ea5feecce..0fb4e58e7 100644
--- a/src/paimon/core/realtime/realtime_append_only_writer.cpp
+++ b/src/paimon/core/realtime/realtime_append_only_writer.cpp
@@ -56,8 +56,8 @@ Result<std::shared_ptr<RealtimeAppendOnlyWriter>> RealtimeAppendOnlyWriter::Crea
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
                            RealtimeContextImpl::Cast(realtime_context));
     RealtimeStoreCreateRequest request{
-        std::move(write_schema),          options, memory_pool, partition, bucket,
-        AppendRealtimeStoreCreateConfig{statistics_mode}};
+        std::move(write_schema),        options,        memory_pool, partition, bucket,
+        RealtimeStoreMode::APPEND_ONLY, statistics_mode};
     PAIMON_ASSIGN_OR_RAISE(RealtimeStoreState store_state,
                            realtime_context_impl->GetOrCreateRealtimeStore(std::move(request)));
     return std::shared_ptr<RealtimeAppendOnlyWriter>(new RealtimeAppendOnlyWriter(
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index 0ea4c61d6..404c0faac 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -48,10 +48,6 @@
 namespace paimon {
 namespace {
 
-bool SameMode(const RealtimeStoreCreateConfig& left, const RealtimeStoreCreateConfig& right) {
-    return left.index() == right.index();
-}
-
 std::string PartitionToString(const std::map<std::string, std::string>& partition) {
     std::string result = "{";
     for (auto iter = partition.begin(); iter != partition.end(); ++iter) {
@@ -122,7 +118,7 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
         initial_offset = offset_iter->second;
     }
     if (iter != stores_.end()) {
-        if (!SameMode(iter->second.mode_config, request.mode_config) ||
+        if (iter->second.mode != request.mode ||
             !iter->second.write_schema->Equals(*requested_schema, /*check_metadata=*/true)) {
             return Status::Invalid("real-time store schema or mode mismatch for partition " +
                                    PartitionToString(key.partition) + ", bucket " +
@@ -151,10 +147,10 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     }
     PAIMON_RETURN_NOT_OK_FROM_ARROW(
         arrow::ExportSchema(*requested_schema, request.write_schema.get()));
-    RealtimeStoreCreateConfig mode_config = request.mode_config;
+    RealtimeStoreMode mode = request.mode;
     Result<std::shared_ptr<RealtimeStore>> store_result = factory_->Create(std::move(request));
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> store, std::move(store_result));
-    stores_.emplace(key, StoreEntry{store, requested_schema, std::move(mode_config)});
+    stores_.emplace(key, StoreEntry{store, requested_schema, mode});
     if (offset_iter != committed_offsets_.end()) {
         reclaimed_offsets_.emplace(key, offset_iter->second);
     }
diff --git a/src/paimon/core/realtime/realtime_context_impl.h b/src/paimon/core/realtime/realtime_context_impl.h
index fd65fc246..29ac7c05d 100644
--- a/src/paimon/core/realtime/realtime_context_impl.h
+++ b/src/paimon/core/realtime/realtime_context_impl.h
@@ -99,7 +99,7 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
     struct StoreEntry {
         std::shared_ptr<RealtimeStore> store;
         std::shared_ptr<arrow::Schema> write_schema;
-        RealtimeStoreCreateConfig mode_config;
+        RealtimeStoreMode mode;
         int64_t materialized_max_sequence_number = -1;
     };
 
diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp
index 538e4c56c..fbdaa86b7 100644
--- a/src/paimon/core/realtime/realtime_context_test.cpp
+++ b/src/paimon/core/realtime/realtime_context_test.cpp
@@ -117,10 +117,11 @@ Result<RealtimeStoreState> GetOrCreateAppendStore(
     const std::shared_ptr<RealtimeContextImpl>& context,
     const std::map<std::string, std::string>& partition, int32_t bucket,
     std::unique_ptr<ArrowSchema> write_schema, const std::map<std::string, std::string>& options,
-    const std::shared_ptr<MemoryPool>& memory_pool) {
+    const std::shared_ptr<MemoryPool>& memory_pool,
+    StatisticsMode statistics_mode = StatisticsMode::NONE) {
     return context->GetOrCreateRealtimeStore(
         RealtimeStoreCreateRequest{std::move(write_schema), options, memory_pool, partition, bucket,
-                                   AppendRealtimeStoreCreateConfig{StatisticsMode::NONE}});
+                                   RealtimeStoreMode::APPEND_ONLY, statistics_mode});
 }
 
 TEST(RealtimeContextTest, TestReusesStoreAndCapturesRegisteredViews) {
@@ -130,9 +131,10 @@ TEST(RealtimeContextTest, TestReusesStoreAndCapturesRegisteredViews) {
                          GetOrCreateAppendStore(context, {{"dt", "2026-08-02"}}, 0,
                                                 MakeWriteSchema(), {{"k", "v"}}, GetDefaultPool()));
     ASSERT_EQ(0, first.initial_offset);
-    ASSERT_OK_AND_ASSIGN(RealtimeStoreState second,
-                         GetOrCreateAppendStore(context, {{"dt", "2026-08-02"}}, 0,
-                                                MakeWriteSchema(), {}, GetDefaultPool()));
+    ASSERT_OK_AND_ASSIGN(
+        RealtimeStoreState second,
+        GetOrCreateAppendStore(context, {{"dt", "2026-08-02"}}, 0, MakeWriteSchema(), {},
+                               GetDefaultPool(), StatisticsMode::FULL));
     ASSERT_EQ(first.store, second.store);
     ASSERT_EQ(0, second.initial_offset);
     ASSERT_EQ(1, factory->stores.size());
@@ -160,6 +162,21 @@ TEST(RealtimeContextTest, TestReusesStoreAndCapturesRegisteredViews) {
     ASSERT_EQ(1, factory->stores[2]->acquire_count);
 }
 
+TEST(RealtimeContextTest, TestRejectsMismatchedModeOnStoreReuse) {
+    auto factory = std::make_shared<TestingRealtimeStoreFactory>();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
+    const std::map<std::string, std::string> partition = {{"dt", "2026-08-02"}};
+    ASSERT_OK(
+        GetOrCreateAppendStore(context, partition, 0, MakeWriteSchema(), {}, GetDefaultPool()));
+
+    ASSERT_NOK_WITH_MSG(
+        context->GetOrCreateRealtimeStore(RealtimeStoreCreateRequest{
+            MakeWriteSchema(), {}, GetDefaultPool(), partition, 0, RealtimeStoreMode::PRIMARY_KEY}),
+        "schema or mode mismatch for partition {dt=2026-08-02}, bucket 0; recreate the "
+        "RealtimeContext");
+    ASSERT_EQ(1, factory->stores.size());
+}
+
 TEST(RealtimeContextTest, TestRejectsMismatchedSchemaOnStoreReuse) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index b04cc8e25..ef85e8b6e 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -288,7 +288,7 @@ Status RealtimePrimaryKeyWriter::FlushSegment(const std::shared_ptr<RealtimeSegm
     PAIMON_ASSIGN_OR_RAISE(
         std::vector<std::unique_ptr<KeyValueRecordReader>> prepared_readers,
         AdaptPreparedCommitBatchReaders(std::move(readers), prepared_schema_, sealed_offsets,
-                                        key_schema_, write_schema_, key_comparator_, memory_pool_));
+                                        key_schema_, write_schema_, memory_pool_));
     std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
     sorted_readers.reserve(prepared_readers.size());
     for (std::unique_ptr<KeyValueRecordReader>& prepared_reader : prepared_readers) {
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index 32231140e..0f6b2189b 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -90,12 +90,11 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateMemoryReaders(
         if (!reader) {
             return Status::Invalid("PK real-time store returned a null query reader");
         }
-        PAIMON_ASSIGN_OR_RAISE(
-            std::unique_ptr<KeyValueRecordReader> prepared_reader,
-            AdaptPreparedBatchReader(
-                std::move(reader), prepared_schema,
-                OffsetRange(split->CommittedEndOffset(), split->MemoryEndOffset()), key_schema,
-                value_schema, key_comparator, memory_pool));
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> prepared_reader,
+                               AdaptPreparedBatchReader(std::move(reader), prepared_schema,
+                                                        OffsetRange(split->CommittedEndOffset(),
+                                                                    split->MemoryEndOffset()),
+                                                        key_schema, value_schema, memory_pool));
         auto merge = std::make_unique<DeduplicateMergeFunction>(false);
         result.push_back(std::make_unique<MergedKeyValueRecordReader>(
             std::move(prepared_reader), key_comparator,
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index ed75b554e..78e8badee 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -538,7 +538,7 @@ class FailAfterPhysicalFileRealtimeStore final : public DelegatingRealtimeStore
     std::shared_ptr<std::atomic<bool>> saw_artifacts_;
 };
 
-enum class CommitReaderMalformation { DROP_LAST, UNSORTED, DUPLICATE_OFFSET, OUT_OF_RANGE_OFFSET };
+enum class CommitReaderMalformation { DROP_LAST, DUPLICATE_OFFSET, OUT_OF_RANGE_OFFSET };
 
 class CorruptingBatchReader final : public BatchReader {
  public:
@@ -550,8 +550,6 @@ class CorruptingBatchReader final : public BatchReader {
         switch (malformation_) {
             case CommitReaderMalformation::DROP_LAST:
                 return DropLast();
-            case CommitReaderMalformation::UNSORTED:
-                return SwapFirstTwo();
             case CommitReaderMalformation::DUPLICATE_OFFSET:
                 return SubstituteOffset(/*offset=*/0);
             case CommitReaderMalformation::OUT_OF_RANGE_OFFSET:
@@ -588,33 +586,6 @@ class CorruptingBatchReader final : public BatchReader {
         return result;
     }
 
-    Result<ReadBatch> SwapFirstTwo() {
-        if (corrupted_) {
-            return delegate_->NextBatch();
-        }
-        corrupted_ = true;
-        PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, delegate_->NextBatch());
-        if (BatchReader::IsEofBatch(batch)) {
-            return MakeEofBatch();
-        }
-        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-            std::shared_ptr<arrow::Array> array,
-            arrow::ImportArray(batch.first.get(), batch.second.get()));
-        if (array->length() < 2) {
-            return Status::Invalid("cannot make a one-row reader unsorted");
-        }
-        arrow::ArrayVector pieces = {array->Slice(1, 1), array->Slice(0, 1)};
-        if (array->length() > 2) {
-            pieces.push_back(array->Slice(2));
-        }
-        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> swapped,
-                                          arrow::Concatenate(pieces));
-        auto output = std::make_unique<ArrowArray>();
-        auto schema = std::make_unique<ArrowSchema>();
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*swapped, output.get(), schema.get()));
-        return ReadBatch(std::move(output), std::move(schema));
-    }
-
     Result<ReadBatch> SubstituteOffset(int64_t offset) {
         PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, delegate_->NextBatch());
         if (BatchReader::IsEofBatch(batch)) {
@@ -652,7 +623,6 @@ class CorruptingBatchReader final : public BatchReader {
 
     std::unique_ptr<BatchReader> delegate_;
     CommitReaderMalformation malformation_;
-    bool corrupted_ = false;
     std::optional<ReadBatch> buffered_;
 };
 
@@ -2399,12 +2369,6 @@ TEST_F(RealtimeWriteInteTest, TestPkRejectsOutOfRangeOffset) {
                                            "offset is outside the sealed range");
 }
 
-TEST_F(RealtimeWriteInteTest, TestPkRejectsUnsortedPluginRows) {
-    CheckPkRejectsCommitReaderMalformation(
-        CommitReaderMalformation::UNSORTED,
-        "not globally sorted by primary key and sequence number");
-}
-
 TEST_F(RealtimeWriteInteTest, TestPkQueryReaderClose) {
     CreatePkTable();
     auto state = std::make_shared<CloseTrackingReaderState>();

From 7a59f9e4329e8539fab7d87a8d3b3f34339ab57a Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Tue, 25 Aug 2026 20:46:12 +0800
Subject: [PATCH 33/62] refactor(realtime): align PK query projection with
 store

---
 include/paimon/realtime/realtime_store.h      |  11 +-
 src/paimon/common/utils/arrow/arrow_utils.cpp |  16 +
 src/paimon/common/utils/arrow/arrow_utils.h   |   3 +
 .../merged_key_value_record_reader_test.cpp   |  46 +--
 .../core/operation/file_store_write.cpp       |   2 +-
 .../key_value_file_store_write_test.cpp       |  36 +-
 .../core/realtime/arrow_realtime_store.cpp    |  25 +-
 .../realtime/prepared_key_value_reader.cpp    | 313 ++++--------------
 .../realtime/primary_key_realtime_store.cpp   | 147 +++++++-
 .../realtime/primary_key_realtime_store.h     |   3 +-
 .../primary_key_realtime_store_test.cpp       | 154 ++++++++-
 .../core/realtime/realtime_context_impl.cpp   |   4 +-
 .../realtime/realtime_primary_key_writer.cpp  |   6 +-
 .../table/source/key_value_table_read.cpp     |   9 +-
 src/paimon/core/table/source/table_scan.cpp   |   2 +-
 15 files changed, 424 insertions(+), 353 deletions(-)

diff --git a/include/paimon/realtime/realtime_store.h b/include/paimon/realtime/realtime_store.h
index 241413b31..fe2b95e1b 100644
--- a/include/paimon/realtime/realtime_store.h
+++ b/include/paimon/realtime/realtime_store.h
@@ -108,7 +108,7 @@ class PAIMON_EXPORT RealtimeReadView {
 /// Parameters used by a `RealtimeStore` to create readers for a query.
 struct PAIMON_EXPORT RealtimeQueryContext {
     /// Append mode receives the requested output fields before the mandatory leading
-    /// `_VALUE_KIND` field is added. Primary-key mode receives the complete prepared schema.
+    /// `_VALUE_KIND` field is added. Primary-key mode receives the requested prepared schema.
     /// This schema is borrowed and remains valid only during `CreateQueryReaders`; plugins must
     /// import or copy it synchronously.
     ::ArrowSchema* read_schema;
@@ -165,10 +165,11 @@ class PAIMON_EXPORT RealtimeStore {
     ///
     /// Append-mode batches contain `_VALUE_KIND` followed by the requested fields except a
     /// duplicate `_VALUE_KIND`, and collectively expose every matching row exactly once.
-    /// Primary-key batches use the prepared transport schema and may contain multiple mutations
-    /// per key; each reader's complete stream is sorted by full primary key then sequence number,
-    /// and the readers collectively expose every raw mutation exactly once. Paimon retains `view`
-    /// for the lifetime of the resulting framework reader.
+    /// Primary-key batches use the requested prepared transport schema, including nested field-ID
+    /// alignment, and may contain multiple mutations per key; each reader's complete stream is
+    /// sorted by full primary key then sequence number, and the readers collectively expose every
+    /// raw mutation exactly once. Paimon retains `view` for the lifetime of the resulting framework
+    /// reader.
     virtual Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
         const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
         const RealtimeQueryContext& context) = 0;
diff --git a/src/paimon/common/utils/arrow/arrow_utils.cpp b/src/paimon/common/utils/arrow/arrow_utils.cpp
index f29e1d11e..34721e125 100644
--- a/src/paimon/common/utils/arrow/arrow_utils.cpp
+++ b/src/paimon/common/utils/arrow/arrow_utils.cpp
@@ -355,6 +355,22 @@ void ArrowUtils::TraverseArray(const std::shared_ptr<arrow::Array>& array) {
     }
 }
 
+uint64_t ArrowUtils::GetArrayMemoryUsage(const std::shared_ptr<arrow::ArrayData>& data) {
+    uint64_t result = 0;
+    for (const std::shared_ptr<arrow::Buffer>& buffer : data->buffers) {
+        if (buffer) {
+            result += static_cast<uint64_t>(buffer->size());
+        }
+    }
+    for (const std::shared_ptr<arrow::ArrayData>& child : data->child_data) {
+        result += GetArrayMemoryUsage(child);
+    }
+    if (data->dictionary) {
+        result += GetArrayMemoryUsage(data->dictionary);
+    }
+    return result;
+}
+
 bool ArrowUtils::EqualsIgnoreNullable(const std::shared_ptr<arrow::DataType>& type,
                                       const std::shared_ptr<arrow::DataType>& other_type) {
     if (type->id() != other_type->id() || type->num_fields() != other_type->num_fields()) {
diff --git a/src/paimon/common/utils/arrow/arrow_utils.h b/src/paimon/common/utils/arrow/arrow_utils.h
index 326b3889e..e8395c4cb 100644
--- a/src/paimon/common/utils/arrow/arrow_utils.h
+++ b/src/paimon/common/utils/arrow/arrow_utils.h
@@ -19,6 +19,7 @@
 
 #pragma once
 
+#include <cstdint>
 #include <vector>
 
 #include "arrow/api.h"
@@ -48,6 +49,8 @@ class PAIMON_EXPORT ArrowUtils {
     // avoid subsequent multi-threading problems.
     static void TraverseArray(const std::shared_ptr<arrow::Array>& array);
 
+    static uint64_t GetArrayMemoryUsage(const std::shared_ptr<arrow::ArrayData>& data);
+
     static Result<std::shared_ptr<arrow::StructArray>> RemoveFieldFromStructArray(
         const std::shared_ptr<arrow::StructArray>& struct_array, const std::string& field_name);
 
diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index 3ba81f03f..c8d80906a 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -325,10 +325,10 @@ TEST_F(MergedKeyValueRecordReaderTest, TestMissingCompositeKey) {
         std::unique_ptr<KeyValueRecordReader> reader,
         AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
                                         arrow::schema({key0, key1}), value_schema, pool_));
-    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "cannot find field id 1");
+    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "field count");
 }
 
-TEST_F(MergedKeyValueRecordReaderTest, TestQueryAddRename) {
+TEST_F(MergedKeyValueRecordReaderTest, TestQueryReaderRequiresStoreAlignedSchema) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Field> old_value = MakeField("old_value", arrow::int32(), 1);
     std::shared_ptr<arrow::Field> renamed_value = MakeField("renamed_value", arrow::int32(), 1);
@@ -347,11 +347,7 @@ TEST_F(MergedKeyValueRecordReaderTest, TestQueryAddRename) {
         std::unique_ptr<KeyValueRecordReader> reader,
         AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
                                         arrow::schema({key}), value_schema, pool_));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader::Iterator> iterator,
-                         reader->NextBatch());
-    ASSERT_OK_AND_ASSIGN(KeyValue key_value, iterator->Next());
-    ASSERT_EQ(20, key_value.value->GetInt(1));
-    ASSERT_TRUE(key_value.value->IsNullAt(2));
+    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "field count");
 }
 
 TEST_F(MergedKeyValueRecordReaderTest, TestMergedReaderErrorRetry) {
@@ -402,35 +398,9 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderSafeDecode) {
         "prepared batch field");
 }
 
-TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderNestedProjection) {
+TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderNestedValues) {
     std::shared_ptr<arrow::Field> id = MakeField("id", arrow::int32(), 0);
-    std::shared_ptr<arrow::Field> item_a = MakeField("a", arrow::int32(), 10);
-    std::shared_ptr<arrow::Field> item_b = MakeField("b", arrow::int32(), 11);
-    std::shared_ptr<arrow::Field> items =
-        MakeField("items", arrow::list(arrow::field("item", arrow::struct_({item_a, item_b}))), 2);
-    std::shared_ptr<arrow::Field> attr_x = MakeField("x", arrow::int32(), 20);
-    std::shared_ptr<arrow::Field> attr_y = MakeField("y", arrow::int32(), 21);
-    std::shared_ptr<arrow::Field> attrs =
-        MakeField("attrs", arrow::map(arrow::utf8(), arrow::struct_({attr_x, attr_y})), 3);
-    std::shared_ptr<arrow::Field> key_left = MakeField("left", arrow::int32(), 30);
-    std::shared_ptr<arrow::Field> key_right = MakeField("right", arrow::int32(), 31);
-    std::shared_ptr<arrow::Field> keyed_values = MakeField(
-        "keyed_values", arrow::map(arrow::struct_({key_left, key_right}), arrow::int32()), 4);
-    std::shared_ptr<arrow::Schema> full_value_schema =
-        arrow::schema({id, items, attrs, keyed_values});
     std::shared_ptr<arrow::Schema> key_schema = arrow::schema({id});
-    std::shared_ptr<arrow::Schema> prepared_schema =
-        MakePreparedSchema(full_value_schema->fields());
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
-        [0, 9, 9, 0, [[1, 2]], [["prefix", [3, 4]]], [[[5, 6], 7]]],
-        [0, 10, 0, 1, [[100, 200], [300, 400]], [["k1", [7, 8]], ["k2", [9, 10]]], [[[11, 12], 13], [[21, 22], 23]]],
-        [0, 11, 11, 2, [[8, 9]], [["suffix", [10, 11]]], [[[12, 13], 14]]]
-    ])")
-            .ValueOrDie());
-    prepared_array = checked_pointer_cast<arrow::StructArray>(prepared_array->Slice(1, 1));
-
     std::shared_ptr<arrow::Field> query_item_b = MakeField("renamed_b", arrow::int32(), 11);
     std::shared_ptr<arrow::Field> query_item_a = MakeField("renamed_a", arrow::int32(), 10);
     std::shared_ptr<arrow::Field> query_items = MakeField(
@@ -448,6 +418,14 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderNestedProjection) {
                   arrow::map(arrow::struct_({query_key_right, query_key_left}), arrow::int32()), 4);
     std::shared_ptr<arrow::Schema> query_value_schema =
         arrow::schema({id, query_items, query_attrs, query_keyed_values});
+    std::shared_ptr<arrow::Schema> prepared_schema =
+        MakePreparedSchema(query_value_schema->fields());
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(
+            prepared_type,
+            R"([[0, 10, 0, 1, [[200, 100], [400, 300]], [["k1", [8, 7]], ["k2", [10, 9]]], [[[12, 11], 13], [[22, 21], 23]]]])")
+            .ValueOrDie();
 
     auto batch_reader = std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
     ASSERT_OK_AND_ASSIGN(
diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp
index 84a324762..6c3b4d033 100644
--- a/src/paimon/core/operation/file_store_write.cpp
+++ b/src/paimon/core/operation/file_store_write.cpp
@@ -198,7 +198,7 @@ Result<std::unique_ptr<FileStoreWrite>> FileStoreWrite::Create(std::unique_ptr<W
     } else {
         // pk table
         if (ctx->GetRealtimeContext()) {
-            PAIMON_RETURN_NOT_OK(ValidatePrimaryKeyRealtimeOptions(options, *schema));
+            PAIMON_RETURN_NOT_OK(PrimaryKeyRealtimeStore::ValidateOptions(options, *schema));
             if (ignore_previous_files) {
                 return Status::NotImplemented(
                     "PK realtime v1 requires restore from the latest snapshot");
diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp
index a2344e803..0f70235a7 100644
--- a/src/paimon/core/operation/key_value_file_store_write_test.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp
@@ -44,6 +44,7 @@
 #include "paimon/common/data/shredding/map_shared_shredding_utils.h"
 #include "paimon/common/data/shredding/map_shredding_defs.h"
 #include "paimon/common/table/special_fields.h"
+#include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/path_util.h"
 #include "paimon/core/io/data_file_meta.h"
@@ -110,7 +111,7 @@ class TestingMemoryPool final : public MemoryPool {
     std::unique_ptr<MemoryPool> delegate_ = GetMemoryPool();
 };
 
-}
+}  // namespace
 
 class KeyValueFileStoreWriteTest : public ::testing::Test {
  protected:
@@ -255,10 +256,23 @@ class KeyValueFileStoreWriteTest : public ::testing::Test {
         if (views.size() != 1) {
             return Status::Invalid("expected exactly one real-time store");
         }
-        RealtimeQueryContext query_context{nullptr, nullptr, false};
-        PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
-                               views[0].store->CreateQueryReaders(
-                                   views[0].read_view, 0, query_context));
+        std::shared_ptr<arrow::Schema> prepared_schema = arrow::schema({
+            DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())
+                ->WithNullable(false),
+            DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
+                ->WithNullable(false),
+            DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()),
+            DataField::ConvertDataFieldToArrowField(
+                DataField(0, arrow::field("id", arrow::int64(), false))),
+            DataField::ConvertDataFieldToArrowField(
+                DataField(1, arrow::field("value", arrow::utf8()))),
+        });
+        auto c_schema = std::make_unique<ArrowSchema>();
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*prepared_schema, c_schema.get()));
+        RealtimeQueryContext query_context{c_schema.get(), nullptr, false};
+        PAIMON_ASSIGN_OR_RAISE(
+            std::vector<std::unique_ptr<BatchReader>> readers,
+            views[0].store->CreateQueryReaders(views[0].read_view, 0, query_context));
         std::vector<std::tuple<int8_t, int64_t, std::string, int64_t, int64_t>> rows;
         for (const std::unique_ptr<BatchReader>& reader : readers) {
             while (true) {
@@ -466,8 +480,8 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeWrite) {
 }
 
 TEST_F(KeyValueFileStoreWriteTest, TestRealtimeOffsetCollision) {
-    const std::map<std::string, std::string> options = {
-        {Options::BUCKET, "1"}, {Options::REALTIME_ENABLED, "true"}};
+    const std::map<std::string, std::string> options = {{Options::BUCKET, "1"},
+                                                        {Options::REALTIME_ENABLED, "true"}};
     const std::shared_ptr<arrow::Schema> schema = arrow::schema({
         arrow::field("id", arrow::int64(), false),
         arrow::field("_REALTIME_OFFSET", arrow::int64()),
@@ -490,8 +504,8 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeOffsetCollision) {
 }
 
 TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) {
-    const std::map<std::string, std::string> options = {
-        {Options::BUCKET, "1"}, {Options::REALTIME_ENABLED, "true"}};
+    const std::map<std::string, std::string> options = {{Options::BUCKET, "1"},
+                                                        {Options::REALTIME_ENABLED, "true"}};
     const std::shared_ptr<arrow::Schema> schema = arrow::schema({
         arrow::field("id", arrow::int64(), false),
         arrow::field("value", arrow::utf8()),
@@ -549,8 +563,8 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) {
 
 TEST_F(KeyValueFileStoreWriteTest, TestRealtimeLimits) {
     const int64_t max = std::numeric_limits<int64_t>::max();
-    const std::map<std::string, std::string> options = {
-        {Options::BUCKET, "1"}, {Options::REALTIME_ENABLED, "true"}};
+    const std::map<std::string, std::string> options = {{Options::BUCKET, "1"},
+                                                        {Options::REALTIME_ENABLED, "true"}};
     const std::shared_ptr<arrow::Schema> schema = arrow::schema({
         arrow::field("id", arrow::int64(), false),
         arrow::field("value", arrow::utf8()),
diff --git a/src/paimon/core/realtime/arrow_realtime_store.cpp b/src/paimon/core/realtime/arrow_realtime_store.cpp
index 18087a29f..1136243e8 100644
--- a/src/paimon/core/realtime/arrow_realtime_store.cpp
+++ b/src/paimon/core/realtime/arrow_realtime_store.cpp
@@ -33,6 +33,7 @@
 #include "paimon/common/reader/complete_row_kind_batch_reader.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/row_kind.h"
+#include "paimon/common/utils/arrow/arrow_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/projected_array.h"
@@ -43,22 +44,6 @@
 namespace paimon {
 namespace {
 
-uint64_t GetArrayMemoryUsage(const std::shared_ptr<arrow::ArrayData>& data) {
-    uint64_t result = 0;
-    for (const std::shared_ptr<arrow::Buffer>& buffer : data->buffers) {
-        if (buffer) {
-            result += static_cast<uint64_t>(buffer->size());
-        }
-    }
-    for (const std::shared_ptr<arrow::ArrayData>& child : data->child_data) {
-        result += GetArrayMemoryUsage(child);
-    }
-    if (data->dictionary) {
-        result += GetArrayMemoryUsage(data->dictionary);
-    }
-    return result;
-}
-
 bool SupportsMinMax(const std::shared_ptr<arrow::DataType>& type) {
     switch (type->id()) {
         case arrow::Type::BOOL:
@@ -393,11 +378,11 @@ Status ArrowRealtimeStore::Write(RealtimeWriteBatch&& write_batch) {
     if (building_range_ && write_batch.offset_range.begin != building_range_->end) {
         return Status::Invalid("real-time offset ranges must be contiguous");
     }
-    uint64_t memory_usage = GetArrayMemoryUsage(struct_array->data());
+    uint64_t memory_usage = ArrowUtils::GetArrayMemoryUsage(struct_array->data());
     if (statistics) {
-        memory_usage += GetArrayMemoryUsage(statistics->min_values->data()) +
-                        GetArrayMemoryUsage(statistics->max_values->data()) +
-                        GetArrayMemoryUsage(statistics->null_counts->data());
+        memory_usage += ArrowUtils::GetArrayMemoryUsage(statistics->min_values->data()) +
+                        ArrowUtils::GetArrayMemoryUsage(statistics->max_values->data()) +
+                        ArrowUtils::GetArrayMemoryUsage(statistics->null_counts->data());
     }
     building_memory_usage_ += memory_usage;
     building_batches_.push_back(
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index a34ccea43..53652cc81 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -23,12 +23,10 @@
 #include <limits>
 #include <memory>
 #include <optional>
-#include <unordered_map>
 #include <utility>
 #include <vector>
 
 #include "arrow/array/array_base.h"
-#include "arrow/array/array_nested.h"
 #include "arrow/array/array_primitive.h"
 #include "arrow/c/bridge.h"
 #include "arrow/type.h"
@@ -39,7 +37,6 @@
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/types/row_kind.h"
 #include "paimon/common/utils/arrow/arrow_utils.h"
-#include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/scope_guard.h"
@@ -66,10 +63,6 @@ void CloseReaders(const std::vector<std::unique_ptr<Reader>>& readers) {
     }
 }
 
-Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
-    const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& read_type,
-    arrow::MemoryPool* arrow_pool);
-
 class RealtimeOffsetCoverage {
  public:
     static Result<std::shared_ptr<RealtimeOffsetCoverage>> Create(const OffsetRange& sealed_offsets,
@@ -158,61 +151,29 @@ Result<int32_t> FindFieldIndexByPaimonId(const arrow::FieldVector& fields, int32
     return Status::Invalid(fmt::format("cannot find field id {} in prepared schema", field_id));
 }
 
-Status ValidateProjectionType(const std::shared_ptr<arrow::DataType>& prepared_type,
-                              const std::shared_ptr<arrow::DataType>& query_type) {
-    if (prepared_type->id() != query_type->id()) {
-        return Status::Invalid(fmt::format("prepared value type {} does not match query type {}",
-                                           prepared_type->ToString(), query_type->ToString()));
-    }
-    switch (query_type->id()) {
-        case arrow::Type::STRUCT: {
-            const arrow::FieldVector& prepared_fields = prepared_type->fields();
-            for (const std::shared_ptr<arrow::Field>& query_field : query_type->fields()) {
-                PAIMON_ASSIGN_OR_RAISE(int32_t query_id,
-                                       NestedProjectionUtils::GetPaimonFieldId(query_field));
-                PAIMON_ASSIGN_OR_RAISE(int32_t prepared_idx,
-                                       FindFieldIndexByPaimonId(prepared_fields, query_id));
-                PAIMON_RETURN_NOT_OK(ValidateProjectionType(prepared_fields[prepared_idx]->type(),
-                                                            query_field->type()));
-            }
-            return Status::OK();
-        }
-        case arrow::Type::LIST:
-            return ValidateProjectionType(prepared_type->field(0)->type(),
-                                          query_type->field(0)->type());
-        case arrow::Type::MAP: {
-            const std::shared_ptr<arrow::MapType> prepared_map =
-                checked_pointer_cast<arrow::MapType>(prepared_type);
-            const std::shared_ptr<arrow::MapType> query_map =
-                checked_pointer_cast<arrow::MapType>(query_type);
-            PAIMON_RETURN_NOT_OK(
-                ValidateProjectionType(prepared_map->key_type(), query_map->key_type()));
-            return ValidateProjectionType(prepared_map->item_type(), query_map->item_type());
-        }
-        default:
-            if (!prepared_type->Equals(*query_type)) {
-                return Status::Invalid(
-                    fmt::format("prepared leaf type {} does not match query type {}",
-                                prepared_type->ToString(), query_type->ToString()));
-            }
-            return Status::OK();
-    }
-}
-
-Status ValidateProjectionSchema(const std::shared_ptr<arrow::Schema>& prepared_schema,
-                                const std::shared_ptr<arrow::Schema>& query_schema) {
+Result<std::vector<int32_t>> ResolveFieldIndexes(
+    const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::shared_ptr<arrow::Schema>& row_schema) {
     arrow::FieldVector prepared_value_fields(
         prepared_schema->fields().begin() + kPreparedValueStartIndex,
         prepared_schema->fields().end());
-    for (const std::shared_ptr<arrow::Field>& query_field : query_schema->fields()) {
-        PAIMON_ASSIGN_OR_RAISE(int32_t query_id,
-                               NestedProjectionUtils::GetPaimonFieldId(query_field));
-        PAIMON_ASSIGN_OR_RAISE(int32_t prepared_idx,
-                               FindFieldIndexByPaimonId(prepared_value_fields, query_id));
-        PAIMON_RETURN_NOT_OK(ValidateProjectionType(prepared_value_fields[prepared_idx]->type(),
-                                                    query_field->type()));
+    std::vector<int32_t> result;
+    result.reserve(row_schema->num_fields());
+    for (const std::shared_ptr<arrow::Field>& row_field : row_schema->fields()) {
+        PAIMON_ASSIGN_OR_RAISE(int32_t field_id,
+                               NestedProjectionUtils::GetPaimonFieldId(row_field));
+        PAIMON_ASSIGN_OR_RAISE(int32_t value_index,
+                               FindFieldIndexByPaimonId(prepared_value_fields, field_id));
+        const std::shared_ptr<arrow::Field>& prepared_field = prepared_value_fields[value_index];
+        if (!prepared_field->type()->Equals(row_field->type())) {
+            return Status::Invalid(fmt::format(
+                "prepared field id {} type {} does not match row "
+                "type {}",
+                field_id, prepared_field->type()->ToString(), row_field->type()->ToString()));
+        }
+        result.push_back(value_index + kPreparedValueStartIndex);
     }
-    return Status::OK();
+    return result;
 }
 
 Status ValidateExactCommitSchema(const std::shared_ptr<arrow::Schema>& prepared_schema,
@@ -229,162 +190,21 @@ Status ValidateExactCommitSchema(const std::shared_ptr<arrow::Schema>& prepared_
     return Status::OK();
 }
 
-Result<std::shared_ptr<arrow::Array>> AlignStructArrayByPaimonIds(
-    const std::shared_ptr<arrow::StructArray>& array,
-    const std::shared_ptr<arrow::StructType>& read_type, arrow::MemoryPool* arrow_pool) {
-    const std::shared_ptr<arrow::StructType> data_type =
-        checked_pointer_cast<arrow::StructType>(array->type());
-    std::unordered_map<int32_t, int32_t> data_field_id_to_idx;
-    data_field_id_to_idx.reserve(data_type->num_fields());
-    for (int32_t i = 0; i < data_type->num_fields(); ++i) {
-        PAIMON_ASSIGN_OR_RAISE(int32_t field_id,
-                               NestedProjectionUtils::GetPaimonFieldId(data_type->field(i)));
-        if (!data_field_id_to_idx.emplace(field_id, i).second) {
-            return Status::Invalid(
-                fmt::format("duplicate field id {} in prepared value struct", field_id));
-        }
-    }
-
-    arrow::ArrayVector aligned_arrays;
-    aligned_arrays.reserve(read_type->num_fields());
-    for (const std::shared_ptr<arrow::Field>& read_field : read_type->fields()) {
-        PAIMON_ASSIGN_OR_RAISE(int32_t read_field_id,
-                               NestedProjectionUtils::GetPaimonFieldId(read_field));
-        auto data_iter = data_field_id_to_idx.find(read_field_id);
-        if (data_iter == data_field_id_to_idx.end()) {
-            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-                std::shared_ptr<arrow::Array> null_child,
-                arrow::MakeArrayOfNull(read_field->type(), array->offset() + array->length(),
-                                       arrow_pool));
-            aligned_arrays.push_back(std::move(null_child));
-            continue;
-        }
-        std::shared_ptr<arrow::Array> child =
-            arrow::MakeArray(array->data()->child_data[data_iter->second]);
-        PAIMON_ASSIGN_OR_RAISE(child, AlignArrayByPaimonIds(child, read_field->type(), arrow_pool));
-        aligned_arrays.push_back(std::move(child));
-    }
-
-    std::shared_ptr<arrow::ArrayData> aligned_data = array->data()->Copy();
-    aligned_data->type = read_type;
-    aligned_data->child_data.clear();
-    aligned_data->child_data.reserve(aligned_arrays.size());
-    for (const std::shared_ptr<arrow::Array>& aligned_array : aligned_arrays) {
-        aligned_data->child_data.push_back(aligned_array->data());
-    }
-    return arrow::MakeArray(std::move(aligned_data));
-}
-
-Result<std::shared_ptr<arrow::Array>> AlignListArrayByPaimonIds(
-    const std::shared_ptr<arrow::ListArray>& array,
-    const std::shared_ptr<arrow::ListType>& read_type, arrow::MemoryPool* arrow_pool) {
-    std::shared_ptr<arrow::Array> values = array->values();
-    PAIMON_ASSIGN_OR_RAISE(values,
-                           AlignArrayByPaimonIds(values, read_type->value_type(), arrow_pool));
-    std::shared_ptr<arrow::ArrayData> new_data = array->data()->Copy();
-    new_data->type = read_type;
-    new_data->child_data = {values->data()};
-    return arrow::MakeArray(new_data);
-}
-
-Result<std::shared_ptr<arrow::Array>> AlignMapArrayByPaimonIds(
-    const std::shared_ptr<arrow::MapArray>& array, const std::shared_ptr<arrow::MapType>& read_type,
-    arrow::MemoryPool* arrow_pool) {
-    std::shared_ptr<arrow::Array> keys = array->keys();
-    PAIMON_ASSIGN_OR_RAISE(keys, AlignArrayByPaimonIds(keys, read_type->key_type(), arrow_pool));
-    std::shared_ptr<arrow::Array> items = array->items();
-    PAIMON_ASSIGN_OR_RAISE(items, AlignArrayByPaimonIds(items, read_type->item_type(), arrow_pool));
-
-    const std::shared_ptr<arrow::ArrayData>& entries_data = array->data()->child_data[0];
-    std::shared_ptr<arrow::ArrayData> new_entries = entries_data->Copy();
-    new_entries->type = arrow::struct_({read_type->key_field(), read_type->item_field()});
-    new_entries->child_data = {keys->data(), items->data()};
-
-    std::shared_ptr<arrow::ArrayData> new_data = array->data()->Copy();
-    new_data->type = read_type;
-    new_data->child_data = {std::move(new_entries)};
-    return arrow::MakeArray(new_data);
-}
-
-Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
-    const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& read_type,
-    arrow::MemoryPool* arrow_pool) {
-    if (array->type()->id() != read_type->id()) {
-        return Status::Invalid(fmt::format("prepared value type {} does not match query type {}",
-                                           array->type()->ToString(), read_type->ToString()));
-    }
-    switch (read_type->id()) {
-        case arrow::Type::STRUCT:
-            return AlignStructArrayByPaimonIds(checked_pointer_cast<arrow::StructArray>(array),
-                                               checked_pointer_cast<arrow::StructType>(read_type),
-                                               arrow_pool);
-        case arrow::Type::LIST:
-            return AlignListArrayByPaimonIds(checked_pointer_cast<arrow::ListArray>(array),
-                                             checked_pointer_cast<arrow::ListType>(read_type),
-                                             arrow_pool);
-        case arrow::Type::MAP:
-            return AlignMapArrayByPaimonIds(checked_pointer_cast<arrow::MapArray>(array),
-                                            checked_pointer_cast<arrow::MapType>(read_type),
-                                            arrow_pool);
-        default:
-            if (!array->type()->Equals(*read_type)) {
-                return Status::Invalid(
-                    fmt::format("prepared leaf type {} does not match query type {}",
-                                array->type()->ToString(), read_type->ToString()));
-            }
-            return array;
-    }
-}
-
-Result<arrow::ArrayVector> ProjectFieldsByPaimonIds(
-    const std::shared_ptr<arrow::StructArray>& data_batch,
-    const std::shared_ptr<arrow::Schema>& prepared_schema,
-    const std::shared_ptr<arrow::Schema>& query_schema, arrow::MemoryPool* arrow_pool) {
-    std::unordered_map<int32_t, int32_t> prepared_field_id_to_idx;
-    prepared_field_id_to_idx.reserve(prepared_schema->num_fields());
-    for (int32_t i = kPreparedValueStartIndex; i < prepared_schema->num_fields(); ++i) {
-        PAIMON_ASSIGN_OR_RAISE(int32_t field_id,
-                               NestedProjectionUtils::GetPaimonFieldId(prepared_schema->field(i)));
-        if (!prepared_field_id_to_idx.emplace(field_id, i).second) {
-            return Status::Invalid(
-                fmt::format("duplicate field id {} in prepared schema", field_id));
-        }
-    }
-
-    arrow::ArrayVector result;
-    result.reserve(query_schema->num_fields());
-    for (const std::shared_ptr<arrow::Field>& query_field : query_schema->fields()) {
-        PAIMON_ASSIGN_OR_RAISE(int32_t query_field_id,
-                               NestedProjectionUtils::GetPaimonFieldId(query_field));
-        auto prepared_iter = prepared_field_id_to_idx.find(query_field_id);
-        if (prepared_iter == prepared_field_id_to_idx.end()) {
-            return Status::Invalid(
-                fmt::format("cannot find field id {} in prepared schema", query_field_id));
-        }
-        std::shared_ptr<arrow::Array> field_array = data_batch->field(prepared_iter->second);
-        PAIMON_ASSIGN_OR_RAISE(field_array,
-                               AlignArrayByPaimonIds(field_array, query_field->type(), arrow_pool));
-        result.push_back(std::move(field_array));
-    }
-    return result;
-}
-
 class PreparedKeyValueReader final : public KeyValueRecordReader {
  public:
     PreparedKeyValueReader(std::unique_ptr<BatchReader>&& reader,
                            const std::shared_ptr<arrow::Schema>& prepared_schema,
                            const std::optional<OffsetRange>& visible_offsets,
-                           const std::shared_ptr<arrow::Schema>& key_schema,
-                           const std::shared_ptr<arrow::Schema>& value_schema,
+                           std::vector<int32_t>&& key_field_indexes,
+                           std::vector<int32_t>&& value_field_indexes,
                            const std::shared_ptr<MemoryPool>& pool,
                            const std::shared_ptr<RealtimeOffsetCoverage>& offset_coverage)
         : reader_(std::move(reader)),
           prepared_schema_(prepared_schema),
           visible_offsets_(visible_offsets),
-          key_schema_(key_schema),
-          value_schema_(value_schema),
+          key_field_indexes_(std::move(key_field_indexes)),
+          value_field_indexes_(std::move(value_field_indexes)),
           pool_(pool),
-          arrow_pool_(GetArrowPool(pool)),
           offset_coverage_(offset_coverage) {}
 
     ~PreparedKeyValueReader() override {
@@ -449,14 +269,21 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
     Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatchImpl() {
         while (true) {
             ResetBatchState();
-            PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader_->NextBatch());
-            if (BatchReader::IsEofBatch(batch)) {
+            BatchReader::ReadBatchWithBitmap batch_with_bitmap;
+            if (visible_offsets_.has_value()) {
+                PAIMON_ASSIGN_OR_RAISE(batch_with_bitmap, reader_->NextBatchWithBitmap());
+            } else {
+                PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader_->NextBatch());
+                batch_with_bitmap.first = std::move(batch);
+            }
+            if (BatchReader::IsEofBatch(batch_with_bitmap)) {
                 if (offset_coverage_ && !offset_coverage_finished_) {
                     offset_coverage_finished_ = true;
                     PAIMON_RETURN_NOT_OK(offset_coverage_->FinishReader());
                 }
                 return std::unique_ptr<KeyValueRecordReader::Iterator>();
             }
+            auto& [batch, selection] = batch_with_bitmap;
             auto& [c_array, c_schema] = batch;
             PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> arrow_array,
                                               arrow::ImportArray(c_array.get(), c_schema.get()));
@@ -473,15 +300,6 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
                     "schema: ",
                     transport_status.ToString());
             }
-            if (visible_offsets_.has_value()) {
-                PAIMON_RETURN_NOT_OK(ValidateProjectionSchema(
-                    arrow::schema(data_batch->type()->fields()), key_schema_));
-                PAIMON_ASSIGN_OR_RAISE(
-                    arrow_array,
-                    AlignArrayByPaimonIds(data_batch, arrow::struct_(prepared_schema_->fields()),
-                                          arrow_pool_.get()));
-                data_batch = checked_pointer_cast<arrow::StructArray>(arrow_array);
-            }
             PAIMON_RETURN_NOT_OK(ValidatePreparedBatch(data_batch));
 
             std::shared_ptr<arrow::NumericArray<arrow::Int64Type>> offset_array =
@@ -495,15 +313,19 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
                 data_batch->field(kValueKindIndex));
             sequence_number_array_ = checked_pointer_cast<arrow::NumericArray<arrow::Int64Type>>(
                 data_batch->field(kSequenceNumberIndex));
-            PAIMON_ASSIGN_OR_RAISE(arrow::ArrayVector key_fields,
-                                   ProjectFieldsByPaimonIds(data_batch, prepared_schema_,
-                                                            key_schema_, arrow_pool_.get()));
-            PAIMON_ASSIGN_OR_RAISE(arrow::ArrayVector value_fields,
-                                   ProjectFieldsByPaimonIds(data_batch, prepared_schema_,
-                                                            value_schema_, arrow_pool_.get()));
+            arrow::ArrayVector key_fields;
+            key_fields.reserve(key_field_indexes_.size());
+            for (int32_t index : key_field_indexes_) {
+                key_fields.push_back(data_batch->field(index));
+            }
+            arrow::ArrayVector value_fields;
+            value_fields.reserve(value_field_indexes_.size());
+            for (int32_t index : value_field_indexes_) {
+                value_fields.push_back(data_batch->field(index));
+            }
             key_ctx_ = std::make_shared<ColumnarBatchContext>(key_fields, pool_);
             value_ctx_ = std::make_shared<ColumnarBatchContext>(value_fields, pool_);
-            if (!SelectVisibleRows(*offset_array)) {
+            if (!SelectRows(*offset_array, std::move(selection))) {
                 continue;
             }
             ArrowUtils::TraverseArray(data_batch);
@@ -544,28 +366,30 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
         return Status::OK();
     }
 
-    bool SelectVisibleRows(const arrow::Int64Array& offsets) {
+    bool SelectRows(const arrow::Int64Array& offsets, RoaringBitmap32&& selection) {
         if (!visible_offsets_.has_value()) {
+            selected_rows_.reserve(offsets.length());
+            for (int64_t row = 0; row < offsets.length(); ++row) {
+                selected_rows_.push_back(row);
+            }
             return true;
         }
-        visible_rows_.emplace();
-        visible_rows_->reserve(offsets.length());
-        for (int64_t row = 0; row < offsets.length(); ++row) {
+        for (auto iter = selection.Begin(); iter != selection.End(); ++iter) {
+            const int32_t row = *iter;
             const int64_t offset = offsets.Value(row);
             if (offset >= visible_offsets_->begin && offset < visible_offsets_->end) {
-                visible_rows_->push_back(row);
+                selected_rows_.push_back(row);
             }
         }
-        return !visible_rows_->empty();
+        return !selected_rows_.empty();
     }
 
     int64_t RowCount() const {
-        return visible_rows_.has_value() ? static_cast<int64_t>(visible_rows_->size())
-                                         : row_kind_array_->length();
+        return static_cast<int64_t>(selected_rows_.size());
     }
 
     int64_t RowAt(int64_t ordinal) const {
-        return visible_rows_.has_value() ? (*visible_rows_)[ordinal] : ordinal;
+        return selected_rows_[ordinal];
     }
 
     void ResetBatchState() {
@@ -573,7 +397,7 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
         value_ctx_.reset();
         row_kind_array_.reset();
         sequence_number_array_.reset();
-        visible_rows_.reset();
+        selected_rows_.clear();
     }
 
  private:
@@ -582,17 +406,16 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
     std::unique_ptr<BatchReader> reader_;
     std::shared_ptr<arrow::Schema> prepared_schema_;
     std::optional<OffsetRange> visible_offsets_;
-    std::shared_ptr<arrow::Schema> key_schema_;
-    std::shared_ptr<arrow::Schema> value_schema_;
+    std::vector<int32_t> key_field_indexes_;
+    std::vector<int32_t> value_field_indexes_;
     std::shared_ptr<MemoryPool> pool_;
-    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
     std::shared_ptr<RealtimeOffsetCoverage> offset_coverage_;
     bool offset_coverage_finished_ = false;
     std::shared_ptr<ColumnarBatchContext> key_ctx_;
     std::shared_ptr<ColumnarBatchContext> value_ctx_;
     std::shared_ptr<arrow::NumericArray<arrow::Int8Type>> row_kind_array_;
     std::shared_ptr<arrow::NumericArray<arrow::Int64Type>> sequence_number_array_;
-    std::optional<std::vector<int64_t>> visible_rows_;
+    std::vector<int64_t> selected_rows_;
 };
 
 }  // namespace
@@ -637,14 +460,16 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderImpl(
     if (!memory_pool) {
         return Status::Invalid("prepared reader memory pool cannot be null");
     }
-    PAIMON_RETURN_NOT_OK(ValidateProjectionSchema(prepared_schema, key_schema));
-    PAIMON_RETURN_NOT_OK(ValidateProjectionSchema(prepared_schema, value_schema));
     if (!visible_offsets.has_value()) {
         PAIMON_RETURN_NOT_OK(ValidateExactCommitSchema(prepared_schema, value_schema));
     }
-    std::unique_ptr<KeyValueRecordReader> result(
-        new PreparedKeyValueReader(std::move(owned_reader), prepared_schema, visible_offsets,
-                                   key_schema, value_schema, memory_pool, offset_coverage));
+    PAIMON_ASSIGN_OR_RAISE(std::vector<int32_t> key_field_indexes,
+                           ResolveFieldIndexes(prepared_schema, key_schema));
+    PAIMON_ASSIGN_OR_RAISE(std::vector<int32_t> value_field_indexes,
+                           ResolveFieldIndexes(prepared_schema, value_schema));
+    std::unique_ptr<KeyValueRecordReader> result(new PreparedKeyValueReader(
+        std::move(owned_reader), prepared_schema, visible_offsets, std::move(key_field_indexes),
+        std::move(value_field_indexes), memory_pool, offset_coverage));
     close_guard.Release();
     return result;
 }
@@ -685,10 +510,10 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> AdaptPreparedCommitBa
                            RealtimeOffsetCoverage::Create(sealed_offsets, readers.size()));
     adapted_readers.reserve(readers.size());
     for (std::unique_ptr<BatchReader>& reader : readers) {
-        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> adapted_reader,
-                               AdaptPreparedBatchReaderImpl(
-                                   std::move(reader), prepared_schema, std::nullopt, key_schema,
-                                   value_schema, memory_pool, offset_coverage));
+        PAIMON_ASSIGN_OR_RAISE(
+            std::unique_ptr<KeyValueRecordReader> adapted_reader,
+            AdaptPreparedBatchReaderImpl(std::move(reader), prepared_schema, std::nullopt,
+                                         key_schema, value_schema, memory_pool, offset_coverage));
         adapted_readers.push_back(std::move(adapted_reader));
     }
     readers_guard.Release();
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index 7d188609d..cfdf88f49 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -20,24 +20,29 @@
 
 #include <mutex>
 #include <optional>
+#include <unordered_map>
 #include <utility>
 #include <vector>
 
 #include "arrow/api.h"
 #include "arrow/c/bridge.h"
+#include "fmt/format.h"
 #include "paimon/common/metrics/metrics_impl.h"
 #include "paimon/common/types/data_field.h"
+#include "paimon/common/utils/arrow/arrow_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/index/pk/primary_key_index_definitions.h"
 #include "paimon/core/realtime/prepared_key_value_reader.h"
 #include "paimon/core/schema/table_schema.h"
+#include "paimon/core/utils/nested_projection_utils.h"
 #include "paimon/macros.h"
 
 namespace paimon {
 
-Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options, const TableSchema& schema) {
+Status PrimaryKeyRealtimeStore::ValidateOptions(const CoreOptions& options,
+                                                const TableSchema& schema) {
     if (options.GetBucket() <= 0) {
         return Status::NotImplemented("PK realtime v1 requires fixed buckets");
     }
@@ -86,20 +91,119 @@ Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options, const Table
 
 namespace {
 
-uint64_t GetArrayMemoryUsage(const std::shared_ptr<arrow::ArrayData>& data) {
-    uint64_t total = 0;
-    for (const std::shared_ptr<arrow::Buffer>& buffer : data->buffers) {
-        if (buffer) {
-            total += static_cast<uint64_t>(buffer->size());
+Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
+    const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& read_type,
+    arrow::MemoryPool* pool);
+
+bool TypesExactlyEqual(const std::shared_ptr<arrow::DataType>& data_type,
+                       const std::shared_ptr<arrow::DataType>& read_type) {
+    if (!data_type->Equals(read_type) || data_type->num_fields() != read_type->num_fields()) {
+        return false;
+    }
+    for (int32_t i = 0; i < data_type->num_fields(); ++i) {
+        if (!data_type->field(i)->Equals(read_type->field(i), /*check_metadata=*/true) ||
+            !TypesExactlyEqual(data_type->field(i)->type(), read_type->field(i)->type())) {
+            return false;
         }
     }
-    for (const std::shared_ptr<arrow::ArrayData>& child : data->child_data) {
-        total += GetArrayMemoryUsage(child);
+    return true;
+}
+
+Result<std::shared_ptr<arrow::Array>> AlignStructArrayByPaimonIds(
+    const std::shared_ptr<arrow::StructArray>& array,
+    const std::shared_ptr<arrow::StructType>& read_type, arrow::MemoryPool* pool) {
+    const std::shared_ptr<arrow::StructType> data_type =
+        checked_pointer_cast<arrow::StructType>(array->type());
+    std::unordered_map<int32_t, int32_t> data_field_indexes;
+    data_field_indexes.reserve(data_type->num_fields());
+    for (int32_t i = 0; i < data_type->num_fields(); ++i) {
+        PAIMON_ASSIGN_OR_RAISE(int32_t field_id,
+                               NestedProjectionUtils::GetPaimonFieldId(data_type->field(i)));
+        if (!data_field_indexes.emplace(field_id, i).second) {
+            return Status::Invalid(fmt::format("duplicate field id {} in stored schema", field_id));
+        }
     }
-    if (data->dictionary) {
-        total += GetArrayMemoryUsage(data->dictionary);
+
+    std::unordered_map<int32_t, bool> requested_field_ids;
+    requested_field_ids.reserve(read_type->num_fields());
+    std::vector<std::shared_ptr<arrow::ArrayData>> children;
+    children.reserve(read_type->num_fields());
+    for (const std::shared_ptr<arrow::Field>& read_field : read_type->fields()) {
+        PAIMON_ASSIGN_OR_RAISE(int32_t field_id,
+                               NestedProjectionUtils::GetPaimonFieldId(read_field));
+        if (!requested_field_ids.emplace(field_id, true).second) {
+            return Status::Invalid(
+                fmt::format("duplicate field id {} in requested schema", field_id));
+        }
+        const auto data_iter = data_field_indexes.find(field_id);
+        if (data_iter == data_field_indexes.end()) {
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+                std::shared_ptr<arrow::Array> null_child,
+                arrow::MakeArrayOfNull(read_field->type(), array->offset() + array->length(),
+                                       pool));
+            children.push_back(null_child->data());
+            continue;
+        }
+        std::shared_ptr<arrow::Array> child =
+            arrow::MakeArray(array->data()->child_data[data_iter->second]);
+        PAIMON_ASSIGN_OR_RAISE(child, AlignArrayByPaimonIds(child, read_field->type(), pool));
+        children.push_back(child->data());
+    }
+
+    std::shared_ptr<arrow::ArrayData> aligned = array->data()->Copy();
+    aligned->type = read_type;
+    aligned->child_data = std::move(children);
+    return arrow::MakeArray(std::move(aligned));
+}
+
+Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
+    const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& read_type,
+    arrow::MemoryPool* pool) {
+    if (TypesExactlyEqual(array->type(), read_type)) {
+        return array;
+    }
+    if (array->type_id() != read_type->id()) {
+        return Status::Invalid(fmt::format("stored value type {} does not match requested type {}",
+                                           array->type()->ToString(), read_type->ToString()));
+    }
+    switch (read_type->id()) {
+        case arrow::Type::STRUCT:
+            return AlignStructArrayByPaimonIds(checked_pointer_cast<arrow::StructArray>(array),
+                                               checked_pointer_cast<arrow::StructType>(read_type),
+                                               pool);
+        case arrow::Type::LIST: {
+            std::shared_ptr<arrow::Array> values =
+                checked_pointer_cast<arrow::ListArray>(array)->values();
+            PAIMON_ASSIGN_OR_RAISE(
+                values, AlignArrayByPaimonIds(values, read_type->field(0)->type(), pool));
+            std::shared_ptr<arrow::ArrayData> aligned = array->data()->Copy();
+            aligned->type = read_type;
+            aligned->child_data = {values->data()};
+            return arrow::MakeArray(std::move(aligned));
+        }
+        case arrow::Type::MAP: {
+            const std::shared_ptr<arrow::MapArray> map =
+                checked_pointer_cast<arrow::MapArray>(array);
+            const std::shared_ptr<arrow::MapType> map_type =
+                checked_pointer_cast<arrow::MapType>(read_type);
+            std::shared_ptr<arrow::Array> keys = map->keys();
+            PAIMON_ASSIGN_OR_RAISE(keys, AlignArrayByPaimonIds(keys, map_type->key_type(), pool));
+            std::shared_ptr<arrow::Array> items = map->items();
+            PAIMON_ASSIGN_OR_RAISE(items,
+                                   AlignArrayByPaimonIds(items, map_type->item_type(), pool));
+            std::shared_ptr<arrow::ArrayData> entries = array->data()->child_data[0]->Copy();
+            entries->type = arrow::struct_({map_type->key_field(), map_type->item_field()});
+            entries->child_data = {keys->data(), items->data()};
+            std::shared_ptr<arrow::ArrayData> aligned = array->data()->Copy();
+            aligned->type = read_type;
+            aligned->child_data = {std::move(entries)};
+            return arrow::MakeArray(std::move(aligned));
+        }
+        default:
+            return Status::Invalid(
+                fmt::format("stored leaf type {} does not match requested type {}",
+                            array->type()->ToString(), read_type->ToString()));
     }
-    return total;
 }
 
 struct StoredBatch {
@@ -201,8 +305,8 @@ class PrimaryKeyRealtimeStore::Impl {
         std::shared_ptr<arrow::StructArray> prepared =
             checked_pointer_cast<arrow::StructArray>(array);
         std::lock_guard<std::mutex> lock(mutex_);
-        building_.push_back(
-            StoredBatch{prepared, write_batch.offset_range, GetArrayMemoryUsage(prepared->data())});
+        building_.push_back(StoredBatch{prepared, write_batch.offset_range,
+                                        ArrowUtils::GetArrayMemoryUsage(prepared->data())});
         building_memory_usage_ += building_.back().memory_usage;
         return Status::OK();
     }
@@ -247,15 +351,28 @@ class PrimaryKeyRealtimeStore::Impl {
     }
 
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
-        const std::shared_ptr<RealtimeReadView>& view, int64_t, const RealtimeQueryContext&) {
+        const std::shared_ptr<RealtimeReadView>& view, int64_t,
+        const RealtimeQueryContext& context) {
         std::shared_ptr<ReadView> typed = std::dynamic_pointer_cast<ReadView>(view);
         if (!typed) {
             return Status::Invalid("read view was not created by the PK real-time store");
         }
+        if (context.read_schema == nullptr || context.read_schema->release == nullptr) {
+            return Status::Invalid("PK real-time query read schema is null");
+        }
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> read_schema,
+                                          arrow::ImportSchema(context.read_schema));
+        PAIMON_RETURN_NOT_OK(ValidatePreparedTransportSchema(read_schema));
         std::vector<std::unique_ptr<BatchReader>> readers;
         for (const std::shared_ptr<Segment>& segment : typed->Segments()) {
             for (const StoredBatch& batch : segment->Batches()) {
-                readers.push_back(std::make_unique<StoredBatchReader>(batch));
+                PAIMON_ASSIGN_OR_RAISE(
+                    std::shared_ptr<arrow::Array> projected,
+                    AlignArrayByPaimonIds(batch.data, arrow::struct_(read_schema->fields()),
+                                          arrow::default_memory_pool()));
+                StoredBatch query_batch{checked_pointer_cast<arrow::StructArray>(projected),
+                                        batch.offset_range, /*memory_usage=*/0};
+                readers.push_back(std::make_unique<StoredBatchReader>(query_batch));
             }
         }
         return readers;
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.h b/src/paimon/core/realtime/primary_key_realtime_store.h
index 35f04485b..df9a5f0c5 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.h
+++ b/src/paimon/core/realtime/primary_key_realtime_store.h
@@ -32,13 +32,12 @@ namespace paimon {
 class CoreOptions;
 class TableSchema;
 
-Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options, const TableSchema& schema);
-
 /// Internal in-memory implementation of the default primary-key `RealtimeStore`.
 class PrimaryKeyRealtimeStore final : public RealtimeStore {
  public:
     static Result<std::shared_ptr<PrimaryKeyRealtimeStore>> Create(
         const std::shared_ptr<arrow::Schema>& prepared_schema);
+    static Status ValidateOptions(const CoreOptions& options, const TableSchema& schema);
 
     ~PrimaryKeyRealtimeStore() override;
 
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index 6ccceba81..405c0d32a 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -30,6 +30,7 @@
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/checked_cast.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/schema/table_schema.h"
 #include "paimon/macros.h"
@@ -39,6 +40,12 @@
 namespace paimon::test {
 namespace {
 
+std::shared_ptr<arrow::Field> FieldWithId(const std::string& name,
+                                          const std::shared_ptr<arrow::DataType>& type,
+                                          int32_t field_id) {
+    return DataField::ConvertDataFieldToArrowField(DataField(field_id, arrow::field(name, type)));
+}
+
 std::shared_ptr<arrow::Schema> PreparedSchema() {
     return arrow::schema(
         {DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
@@ -93,6 +100,18 @@ std::unique_ptr<RecordBatch> MakeBatch(const std::shared_ptr<arrow::Schema>& sch
     return RecordBatchBuilder(c_array.get()).Finish().value();
 }
 
+std::unique_ptr<RecordBatch> MakeSlicedBatch(const std::shared_ptr<arrow::Schema>& schema,
+                                             const std::string& json, int64_t offset,
+                                             int64_t length) {
+    std::shared_ptr<arrow::Array> array =
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema->fields()), json)
+            .ValueOrDie()
+            ->Slice(offset, length);
+    auto c_array = std::make_unique<ArrowArray>();
+    EXPECT_TRUE(arrow::ExportArray(*array, c_array.get()).ok());
+    return RecordBatchBuilder(c_array.get()).Finish().value();
+}
+
 void AssertOffsetsZero(const ArrowArray* array) {
     ASSERT_NE(nullptr, array);
     ASSERT_EQ(0, array->offset);
@@ -125,7 +144,7 @@ Result<std::string> ReadJson(const std::vector<std::unique_ptr<BatchReader>>& re
 
 TEST(PrimaryKeyRealtimeStoreOptionsTest, TestSupportedOptions) {
     ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
-    ASSERT_OK(ValidatePrimaryKeyRealtimeOptions(options, *PkSchema()));
+    ASSERT_OK(PrimaryKeyRealtimeStore::ValidateOptions(options, *PkSchema()));
 }
 
 TEST(PrimaryKeyRealtimeStoreOptionsTest, TestUnsupportedOptions) {
@@ -143,16 +162,18 @@ TEST(PrimaryKeyRealtimeStoreOptionsTest, TestUnsupportedOptions) {
     };
     for (const std::map<std::string, std::string>& option_map : unsupported_options) {
         ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
-        ASSERT_NOK(ValidatePrimaryKeyRealtimeOptions(options, *PkSchema()));
+        ASSERT_NOK(PrimaryKeyRealtimeStore::ValidateOptions(options, *PkSchema()));
     }
 }
 
 TEST(PrimaryKeyRealtimeStoreOptionsTest, TestRejectsFloatingPrimaryKeys) {
     ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
-    ASSERT_NOK_WITH_MSG(ValidatePrimaryKeyRealtimeOptions(options, *PkSchema(arrow::float32())),
-                        "FLOAT or DOUBLE primary keys");
-    ASSERT_NOK_WITH_MSG(ValidatePrimaryKeyRealtimeOptions(options, *PkSchema(arrow::float64())),
-                        "FLOAT or DOUBLE primary keys");
+    ASSERT_NOK_WITH_MSG(
+        PrimaryKeyRealtimeStore::ValidateOptions(options, *PkSchema(arrow::float32())),
+        "FLOAT or DOUBLE primary keys");
+    ASSERT_NOK_WITH_MSG(
+        PrimaryKeyRealtimeStore::ValidateOptions(options, *PkSchema(arrow::float64())),
+        "FLOAT or DOUBLE primary keys");
 }
 
 TEST(PrimaryKeyRealtimeStoreOptionsTest, TestRejectsEnabledGlobalIndex) {
@@ -160,7 +181,7 @@ TEST(PrimaryKeyRealtimeStoreOptionsTest, TestRejectsEnabledGlobalIndex) {
                                                            {Options::PK_BTREE_INDEX_COLUMNS, "id"}};
     ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
     ASSERT_NOK_WITH_MSG(
-        ValidatePrimaryKeyRealtimeOptions(options, *PkSchema(arrow::int64(), option_map)),
+        PrimaryKeyRealtimeStore::ValidateOptions(options, *PkSchema(arrow::int64(), option_map)),
         "does not support global indexes");
 }
 
@@ -306,7 +327,9 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderPerStoredBatch) {
     ASSERT_OK(
         store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 2, 1, 1, "one"]])"), OffsetRange(1, 2)}));
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
-    RealtimeQueryContext context{/*read_schema=*/nullptr, /*predicate=*/nullptr,
+    auto c_schema = std::make_unique<ArrowSchema>();
+    ASSERT_TRUE(arrow::ExportSchema(*PreparedSchema(), c_schema.get()).ok());
+    RealtimeQueryContext context{/*read_schema=*/c_schema.get(), /*predicate=*/nullptr,
                                  /*enable_predicate_pushdown=*/false};
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
                          store->CreateQueryReaders(view, /*offset_begin=*/0, context));
@@ -316,5 +339,120 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderPerStoredBatch) {
     ASSERT_NE(std::string::npos, actual.find("\"two\""));
 }
 
+TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderProjectsTopLevelFieldsById) {
+    const std::shared_ptr<arrow::Schema> stored_schema = PreparedSchema();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                         PrimaryKeyRealtimeStore::Create(stored_schema));
+    ASSERT_OK(store->Write(RealtimeWriteBatch{
+        MakeSlicedBatch(stored_schema,
+                        R"([[0, 1, 0, 6, "six"], [0, 2, 1, 7, "seven"], [0, 3, 2, 8, "eight"]])", 1,
+                        1),
+        OffsetRange(0, 1)}));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
+
+    arrow::FieldVector requested_fields(stored_schema->fields().begin(),
+                                        stored_schema->fields().begin() + 3);
+    requested_fields.push_back(FieldWithId("renamed_value", arrow::utf8(), 1));
+    requested_fields.push_back(FieldWithId("added", arrow::int32(), 2));
+    std::shared_ptr<arrow::Schema> requested_schema = arrow::schema(std::move(requested_fields));
+    auto c_schema = std::make_unique<ArrowSchema>();
+    ASSERT_TRUE(arrow::ExportSchema(*requested_schema, c_schema.get()).ok());
+    RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store->CreateQueryReaders(view, /*offset_begin=*/0, context));
+    ASSERT_EQ(1, readers.size());
+    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, readers[0]->NextBatch());
+    arrow::Result<std::shared_ptr<arrow::Array>> import_result =
+        arrow::ImportArray(batch.first.get(), batch.second.get());
+    ASSERT_TRUE(import_result.ok()) << import_result.status().ToString();
+    std::shared_ptr<arrow::Array> array = std::move(import_result).ValueOrDie();
+    ASSERT_TRUE(array->type()->Equals(arrow::struct_(requested_schema->fields())));
+    std::shared_ptr<arrow::StructArray> projected = checked_pointer_cast<arrow::StructArray>(array);
+    ASSERT_EQ(5, projected->num_fields());
+    ASSERT_EQ("seven", checked_pointer_cast<arrow::StringArray>(projected->field(3))->GetString(0));
+    ASSERT_TRUE(projected->field(4)->IsNull(0));
+}
+
+TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderAlignsNestedFieldsById) {
+    const std::shared_ptr<arrow::Field> stored_a = FieldWithId("a", arrow::int32(), 10);
+    const std::shared_ptr<arrow::Field> stored_b = FieldWithId("b", arrow::int32(), 11);
+    const std::shared_ptr<arrow::Field> stored_x = FieldWithId("x", arrow::int32(), 20);
+    const std::shared_ptr<arrow::Field> stored_y = FieldWithId("y", arrow::int32(), 21);
+    arrow::FieldVector stored_fields = {
+        DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
+        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
+            ->WithNullable(false),
+        DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()),
+        FieldWithId("id", arrow::int64(), 0),
+        FieldWithId("items", arrow::list(arrow::struct_({stored_a, stored_b})), 1),
+        FieldWithId("attrs", arrow::map(arrow::utf8(), arrow::struct_({stored_x, stored_y})), 2)};
+    std::shared_ptr<arrow::Schema> stored_schema = arrow::schema(std::move(stored_fields));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                         PrimaryKeyRealtimeStore::Create(stored_schema));
+    ASSERT_OK(store->Write(RealtimeWriteBatch{
+        MakeSlicedBatch(
+            stored_schema,
+            R"([[0, 1, 0, 6, [[1, 2]], [["before", [3, 4]]]], [0, 2, 1, 7, [[100, 200], null], [["k1", [7, 8]], ["k2", null]]], [0, 3, 2, 8, [[9, 10]], [["after", [11, 12]]]]])",
+            1, 1),
+        OffsetRange(0, 1)}));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
+
+    const std::shared_ptr<arrow::Field> requested_b = FieldWithId("renamed_b", arrow::int32(), 11);
+    const std::shared_ptr<arrow::Field> requested_a = FieldWithId("renamed_a", arrow::int32(), 10);
+    const std::shared_ptr<arrow::Field> requested_item_missing =
+        FieldWithId("added_item", arrow::int32(), 12);
+    const std::shared_ptr<arrow::Field> requested_y = FieldWithId("renamed_y", arrow::int32(), 21);
+    const std::shared_ptr<arrow::Field> requested_x = FieldWithId("renamed_x", arrow::int32(), 20);
+    const std::shared_ptr<arrow::Field> requested_attr_missing =
+        FieldWithId("added_attr", arrow::int32(), 22);
+    arrow::FieldVector requested_fields(stored_schema->fields().begin(),
+                                        stored_schema->fields().begin() + 3);
+    requested_fields.push_back(FieldWithId(
+        "renamed_items",
+        arrow::list(arrow::struct_({requested_b, requested_item_missing, requested_a})), 1));
+    requested_fields.push_back(
+        FieldWithId("renamed_attrs",
+                    arrow::map(arrow::utf8(),
+                               arrow::struct_({requested_y, requested_attr_missing, requested_x})),
+                    2));
+    std::shared_ptr<arrow::Schema> requested_schema = arrow::schema(std::move(requested_fields));
+    auto c_schema = std::make_unique<ArrowSchema>();
+    ASSERT_TRUE(arrow::ExportSchema(*requested_schema, c_schema.get()).ok());
+    RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store->CreateQueryReaders(view, /*offset_begin=*/0, context));
+    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, readers[0]->NextBatch());
+    arrow::Result<std::shared_ptr<arrow::Array>> import_result =
+        arrow::ImportArray(batch.first.get(), batch.second.get());
+    ASSERT_TRUE(import_result.ok()) << import_result.status().ToString();
+    std::shared_ptr<arrow::Array> array = std::move(import_result).ValueOrDie();
+    ASSERT_TRUE(array->type()->Equals(arrow::struct_(requested_schema->fields())));
+    std::shared_ptr<arrow::StructArray> projected = checked_pointer_cast<arrow::StructArray>(array);
+    const std::shared_ptr<arrow::ListArray> items =
+        checked_pointer_cast<arrow::ListArray>(projected->field(3));
+    const std::shared_ptr<arrow::StructArray> item_values =
+        checked_pointer_cast<arrow::StructArray>(items->value_slice(0));
+    ASSERT_EQ(200, checked_pointer_cast<arrow::Int32Array>(item_values->field(0))->Value(0));
+    ASSERT_TRUE(item_values->field(1)->IsNull(0));
+    ASSERT_EQ(100, checked_pointer_cast<arrow::Int32Array>(item_values->field(2))->Value(0));
+    ASSERT_TRUE(item_values->IsNull(1));
+
+    const std::shared_ptr<arrow::MapArray> attrs =
+        checked_pointer_cast<arrow::MapArray>(projected->field(4));
+    const int64_t attr_offset = attrs->value_offset(0);
+    const int64_t attr_length = attrs->value_length(0);
+    const std::shared_ptr<arrow::StringArray> attr_keys =
+        checked_pointer_cast<arrow::StringArray>(attrs->keys()->Slice(attr_offset, attr_length));
+    ASSERT_EQ("k1", attr_keys->GetString(0));
+    const std::shared_ptr<arrow::StructArray> attr_values =
+        checked_pointer_cast<arrow::StructArray>(attrs->items()->Slice(attr_offset, attr_length));
+    ASSERT_EQ(8, checked_pointer_cast<arrow::Int32Array>(attr_values->field(0))->Value(0));
+    ASSERT_TRUE(attr_values->field(1)->IsNull(0));
+    ASSERT_EQ(7, checked_pointer_cast<arrow::Int32Array>(attr_values->field(2))->Value(0));
+    ASSERT_TRUE(attr_values->IsNull(1));
+}
+
 }  // namespace
 }  // namespace paimon::test
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index 404c0faac..052b616f6 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -148,8 +148,8 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     PAIMON_RETURN_NOT_OK_FROM_ARROW(
         arrow::ExportSchema(*requested_schema, request.write_schema.get()));
     RealtimeStoreMode mode = request.mode;
-    Result<std::shared_ptr<RealtimeStore>> store_result = factory_->Create(std::move(request));
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> store, std::move(store_result));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> store,
+                           factory_->Create(std::move(request)));
     stores_.emplace(key, StoreEntry{store, requested_schema, mode});
     if (offset_iter != committed_offsets_.end()) {
         reclaimed_offsets_.emplace(key, offset_iter->second);
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index ef85e8b6e..bcc8f6705 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -246,10 +246,8 @@ Status RealtimePrimaryKeyWriter::Write(std::unique_ptr<RecordBatch>&& batch) {
         std::move(prepared_batch), OffsetRange(next_offset_, next_offset_ + count)}));
     next_offset_ += count;
     last_sequence_number_ += count;
-    PAIMON_RETURN_NOT_OK(
-        realtime_context_
-            ->AdvanceMaterializedMaxSequenceNumber(partition_bucket_, last_sequence_number_)
-            .status());
+    PAIMON_RETURN_NOT_OK(realtime_context_->AdvanceMaterializedMaxSequenceNumber(
+        partition_bucket_, last_sequence_number_));
     return Status::OK();
 }
 
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index 0f6b2189b..64b866439 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -59,17 +59,14 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateMemoryReaders(
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
     const std::shared_ptr<FieldsComparator>& key_comparator,
-    const std::shared_ptr<InternalReadContext>& context,
     const std::shared_ptr<MemoryPool>& memory_pool) {
-    std::shared_ptr<arrow::Schema> full_value_schema =
-        DataField::ConvertDataFieldsToArrowSchema(context->GetTableSchema()->Fields());
     arrow::FieldVector prepared_fields = {
         DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
         DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
             ->WithNullable(false),
         DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset())};
-    prepared_fields.insert(prepared_fields.end(), full_value_schema->fields().begin(),
-                           full_value_schema->fields().end());
+    prepared_fields.insert(prepared_fields.end(), value_schema->fields().begin(),
+                           value_schema->fields().end());
     std::shared_ptr<arrow::Schema> prepared_schema = arrow::schema(std::move(prepared_fields));
     auto c_schema = std::make_unique<ArrowSchema>();
     PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*prepared_schema, c_schema.get()));
@@ -276,7 +273,7 @@ Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateRealtimeReader(
                 std::vector<std::unique_ptr<KeyValueRecordReader>> memory_readers,
                 CreateMemoryReaders(realtime_split, memory, merge_read->GetKeySchema(),
                                     merge_read->GetValueSchema(), merge_read->GetKeyComparator(),
-                                    context_, GetMemoryPool()));
+                                    GetMemoryPool()));
             PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> reader,
                                    merge_read->CreateRealtimeReader(realtime_split->DiskSplits(),
                                                                     std::move(memory_readers)));
diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp
index 7f9fe568b..70ab0cb8e 100644
--- a/src/paimon/core/table/source/table_scan.cpp
+++ b/src/paimon/core/table/source/table_scan.cpp
@@ -237,7 +237,7 @@ Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions&
             return Status::NotImplemented(
                 "PK real-time union read does not support read-optimized scans");
         }
-        PAIMON_RETURN_NOT_OK(ValidatePrimaryKeyRealtimeOptions(core_options, table_schema));
+        PAIMON_RETURN_NOT_OK(PrimaryKeyRealtimeStore::ValidateOptions(core_options, table_schema));
     }
     if (context.IsStreamingMode()) {
         return Status::Invalid("real-time union read currently supports batch scans only");

From 884270fed6321944cd1c2dd0407482615369b7a7 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Tue, 25 Aug 2026 22:24:26 +0800
Subject: [PATCH 34/62] refactor(realtime): address review feedback

---
 include/paimon/realtime/realtime_store.h      |   6 +-
 src/paimon/common/table/special_fields.h      |   2 +-
 .../common/table/special_fields_test.cpp      |   2 +-
 .../merged_key_value_record_reader_test.cpp   |   4 +-
 .../core/mergetree/merge_tree_writer.cpp      |   4 +-
 src/paimon/core/mergetree/merge_tree_writer.h |   3 +-
 .../core/mergetree/merge_tree_writer_test.cpp |  16 +-
 .../core/operation/file_store_write.cpp       |  47 +++---
 .../operation/key_value_file_store_write.cpp  |   3 +-
 .../core/operation/merge_file_split_read.cpp  | 148 ++++++++++--------
 .../core/operation/merge_file_split_read.h    |  14 ++
 .../realtime/arrow_realtime_store_test.cpp    |  12 +-
 .../realtime/prepared_key_value_reader.cpp    |  14 +-
 .../core/realtime/prepared_key_value_reader.h |  40 ++---
 .../realtime/primary_key_realtime_store.cpp   |   4 +-
 .../realtime/realtime_append_only_writer.cpp  |   8 +-
 .../core/realtime/realtime_context_impl.cpp   |  16 +-
 .../core/realtime/realtime_context_impl.h     |   3 +-
 .../core/realtime/realtime_context_test.cpp   |  11 +-
 .../realtime/realtime_primary_key_writer.cpp  |  10 +-
 .../core/schema/schema_validation_test.cpp    |   5 +-
 .../table/source/key_value_table_read.cpp     |  22 ++-
 test/inte/realtime_write_inte_test.cpp        |  21 ++-
 23 files changed, 233 insertions(+), 182 deletions(-)

diff --git a/include/paimon/realtime/realtime_store.h b/include/paimon/realtime/realtime_store.h
index fe2b95e1b..cbfe96595 100644
--- a/include/paimon/realtime/realtime_store.h
+++ b/include/paimon/realtime/realtime_store.h
@@ -57,10 +57,6 @@ struct PAIMON_EXPORT RealtimeStoreCreateRequest {
     std::map<std::string, std::string> options;
     /// Memory pool for allocations retained by the store.
     std::shared_ptr<MemoryPool> memory_pool;
-    /// Partition values identifying the store.
-    std::map<std::string, std::string> partition;
-    /// Bucket identifying the store within its partition.
-    int32_t bucket = -1;
     /// Table mode implemented by the store.
     RealtimeStoreMode mode = RealtimeStoreMode::APPEND_ONLY;
     /// Statistics collected by append-only stores.
@@ -192,7 +188,7 @@ class PAIMON_EXPORT RealtimeStoreFactory {
  public:
     virtual ~RealtimeStoreFactory() = default;
 
-    /// Creates a store for the requested table mode and partition-bucket.
+    /// Creates a store for the requested table mode.
     /// The factory consumes `request`, including ownership of `request.write_schema`.
     virtual Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) = 0;
 };
diff --git a/src/paimon/common/table/special_fields.h b/src/paimon/common/table/special_fields.h
index 3279bfed6..9771ac232 100644
--- a/src/paimon/common/table/special_fields.h
+++ b/src/paimon/common/table/special_fields.h
@@ -79,7 +79,7 @@ struct SpecialFields {
         }
         return field_name == SequenceNumber().Name() || field_name == ValueKind().Name() ||
                field_name == RowKind().Name() || field_name == RowId().Name() ||
-               field_name == IndexScore().Name();
+               field_name == IndexScore().Name() || field_name == RealtimeOffset().Name();
     }
 
     // TODO(xinyu.lxy): add a func to complete row-tracking fields
diff --git a/src/paimon/common/table/special_fields_test.cpp b/src/paimon/common/table/special_fields_test.cpp
index b61d289b0..58a025ba2 100644
--- a/src/paimon/common/table/special_fields_test.cpp
+++ b/src/paimon/common/table/special_fields_test.cpp
@@ -73,7 +73,7 @@ TEST(SpecialFieldsTest, TestIsSystemField) {
     ASSERT_TRUE(SpecialFields::IsSystemField("rowkind"));
     ASSERT_TRUE(SpecialFields::IsSystemField("_ROW_ID"));
     ASSERT_TRUE(SpecialFields::IsSystemField("_INDEX_SCORE"));
-    ASSERT_FALSE(SpecialFields::IsSystemField("_REALTIME_OFFSET"));
+    ASSERT_TRUE(SpecialFields::IsSystemField("_REALTIME_OFFSET"));
     ASSERT_TRUE(SpecialFields::IsSystemField("_KEY_0"));
 }
 
diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index c8d80906a..d86e3fc9a 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -71,8 +71,8 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderForTest(
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
     const std::shared_ptr<MemoryPool>& memory_pool) {
-    return AdaptPreparedBatchReader(std::move(reader), prepared_schema, visible_offsets, key_schema,
-                                    value_schema, memory_pool);
+    return PreparedKeyValueReaderFactory::Create(
+        std::move(reader), prepared_schema, visible_offsets, key_schema, value_schema, memory_pool);
 }
 
 class TrackingBatchReader : public BatchReader {
diff --git a/src/paimon/core/mergetree/merge_tree_writer.cpp b/src/paimon/core/mergetree/merge_tree_writer.cpp
index 49961536a..bcf98e9f7 100644
--- a/src/paimon/core/mergetree/merge_tree_writer.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer.cpp
@@ -154,7 +154,7 @@ Status MergeTreeWriter::Write(std::unique_ptr<RecordBatch>&& moved_batch) {
     return Status::OK();
 }
 
-Status MergeTreeWriter::WriteSortedReaders(
+Status MergeTreeWriter::WriteSortedReadersToFiles(
     std::vector<std::unique_ptr<KeyValueRecordReader>>&& readers) {
     auto raw_readers_guard = ScopeGuard([&]() -> void {
         for (std::unique_ptr<KeyValueRecordReader>& reader : readers) {
@@ -311,7 +311,7 @@ Status MergeTreeWriter::FlushWriteBuffer(bool wait_for_latest_compaction,
         auto cleanup_guard = ScopeGuard([&]() { write_buffer_->Clear(); });
         PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
                                write_buffer_->CreateReaders());
-        PAIMON_RETURN_NOT_OK(WriteSortedReaders(std::move(readers)));
+        PAIMON_RETURN_NOT_OK(WriteSortedReadersToFiles(std::move(readers)));
     }
     PAIMON_RETURN_NOT_OK(TrySyncLatestCompaction(wait_for_latest_compaction));
     PAIMON_RETURN_NOT_OK(compact_manager_->TriggerCompaction(forced_full_compaction));
diff --git a/src/paimon/core/mergetree/merge_tree_writer.h b/src/paimon/core/mergetree/merge_tree_writer.h
index 01efd975c..17a9dc51c 100644
--- a/src/paimon/core/mergetree/merge_tree_writer.h
+++ b/src/paimon/core/mergetree/merge_tree_writer.h
@@ -72,7 +72,8 @@ class MergeTreeWriter : public BatchWriter {
 
     /// Consumes readers whose complete streams are individually sorted by primary key and sequence
     /// number. Readers are closed on success or failure.
-    Status WriteSortedReaders(std::vector<std::unique_ptr<KeyValueRecordReader>>&& readers);
+    Status WriteSortedReadersToFiles(
+        std::vector<std::unique_ptr<KeyValueRecordReader>>&& readers);
 
     Status Compact(bool full_compaction) override;
 
diff --git a/src/paimon/core/mergetree/merge_tree_writer_test.cpp b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
index 9ce5498cb..e93935a41 100644
--- a/src/paimon/core/mergetree/merge_tree_writer_test.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
@@ -359,7 +359,7 @@ TEST_P(MergeTreeWriterTest, TestSimple) {
                          CreateMergeWriter(-1, dir->Str(), sorted_reader_path_factory, 1, options));
     std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
     sorted_readers.push_back(CreateSingleReader(sorted_reader_array));
-    ASSERT_OK(sorted_reader_writer->WriteSortedReaders(std::move(sorted_readers)));
+    ASSERT_OK(sorted_reader_writer->WriteSortedReadersToFiles(std::move(sorted_readers)));
     ASSERT_OK_AND_ASSIGN(CommitIncrement sorted_reader_commit_increment,
                          sorted_reader_writer->PrepareCommit(false));
     ASSERT_OK(sorted_reader_writer->Close());
@@ -464,7 +464,7 @@ TEST_P(MergeTreeWriterTest, TestWriteMultiBatch) {
                          CreateMergeWriter(9, dir->Str(), sorted_reader_path_factory, 0, options));
     std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
     sorted_readers.push_back(CreateSingleReader(sorted_reader_array));
-    ASSERT_OK(sorted_reader_writer->WriteSortedReaders(std::move(sorted_readers)));
+    ASSERT_OK(sorted_reader_writer->WriteSortedReadersToFiles(std::move(sorted_readers)));
     ASSERT_OK_AND_ASSIGN(CommitIncrement sorted_reader_commit_increment,
                          sorted_reader_writer->PrepareCommit(false));
     ASSERT_OK(sorted_reader_writer->Close());
@@ -498,7 +498,7 @@ TEST_P(MergeTreeWriterTest, TestSortedReaders) {
     std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
     sorted_readers.push_back(CreateSingleReader(sorted_reader_array));
 
-    ASSERT_OK(merge_writer->WriteSortedReaders(std::move(sorted_readers)));
+    ASSERT_OK(merge_writer->WriteSortedReadersToFiles(std::move(sorted_readers)));
     ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment, merge_writer->PrepareCommit(false));
     ASSERT_OK(merge_writer->Close());
 
@@ -564,7 +564,7 @@ TEST_P(MergeTreeWriterTest, TestMergeSortedReaders) {
     sorted_readers.push_back(std::make_unique<TrackingKeyValueRecordReader>(
         CreateSingleReader(second_array), &second_closed));
 
-    ASSERT_OK(merge_writer->WriteSortedReaders(std::move(sorted_readers)));
+    ASSERT_OK(merge_writer->WriteSortedReadersToFiles(std::move(sorted_readers)));
     ASSERT_TRUE(first_closed);
     ASSERT_TRUE(second_closed);
     ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment, merge_writer->PrepareCommit(false));
@@ -611,7 +611,7 @@ TEST_P(MergeTreeWriterTest, TestSortedReaderOwnership) {
     sorted_readers.push_back(std::make_unique<TrackingKeyValueRecordReader>(
         CreateSingleReader(sorted_reader_array), &closed));
 
-    ASSERT_OK(merge_writer->WriteSortedReaders(std::move(sorted_readers)));
+    ASSERT_OK(merge_writer->WriteSortedReadersToFiles(std::move(sorted_readers)));
     ASSERT_TRUE(closed);
     ASSERT_OK(merge_writer->Close());
 }
@@ -629,12 +629,12 @@ TEST_P(MergeTreeWriterTest, TestSortedReaderFailure) {
                          CreateMergeWriter(-1, dir->Str(), path_factory, 0, options));
 
     std::vector<std::unique_ptr<KeyValueRecordReader>> empty_readers;
-    Status empty_status = merge_writer->WriteSortedReaders(std::move(empty_readers));
+    Status empty_status = merge_writer->WriteSortedReadersToFiles(std::move(empty_readers));
     ASSERT_TRUE(empty_status.IsInvalid());
 
     std::vector<std::unique_ptr<KeyValueRecordReader>> null_readers;
     null_readers.push_back(nullptr);
-    Status null_status = merge_writer->WriteSortedReaders(std::move(null_readers));
+    Status null_status = merge_writer->WriteSortedReadersToFiles(std::move(null_readers));
     ASSERT_TRUE(null_status.IsInvalid());
 
     auto sorted_reader_array = std::dynamic_pointer_cast<arrow::StructArray>(
@@ -648,7 +648,7 @@ TEST_P(MergeTreeWriterTest, TestSortedReaderFailure) {
     failing_readers.push_back(std::make_unique<TrackingKeyValueRecordReader>(
         CreateSingleReader(sorted_reader_array, /*batch_size=*/16, expected_status),
         &failing_reader_closed));
-    Status failing_status = merge_writer->WriteSortedReaders(std::move(failing_readers));
+    Status failing_status = merge_writer->WriteSortedReadersToFiles(std::move(failing_readers));
     ASSERT_EQ(expected_status, failing_status);
     ASSERT_TRUE(failing_reader_closed);
     ASSERT_OK(merge_writer->Close());
diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp
index 6c3b4d033..26befb7ee 100644
--- a/src/paimon/core/operation/file_store_write.cpp
+++ b/src/paimon/core/operation/file_store_write.cpp
@@ -58,6 +58,27 @@ struct KeyValue;
 template <typename T>
 class MergeFunctionWrapper;
 
+namespace {
+
+Status RestoreRealtimeCommittedProgress(const std::shared_ptr<RealtimeContext>& realtime_context,
+                                        const std::shared_ptr<SnapshotManager>& snapshot_manager,
+                                        const CoreOptions& options) {
+    PAIMON_ASSIGN_OR_RAISE(std::optional<Snapshot> latest_snapshot,
+                           snapshot_manager->LatestSnapshot());
+    if (latest_snapshot) {
+        PAIMON_ASSIGN_OR_RAISE(
+            RealtimeOffsetMap realtime_committed_offsets,
+            RealtimeCommitProperties::ReadOffsets(latest_snapshot, options.GetFileSystem()));
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
+                               RealtimeContextImpl::Cast(realtime_context));
+        PAIMON_RETURN_NOT_OK(realtime_context_impl->AdvanceCommittedProgress(
+            latest_snapshot->Id(), realtime_committed_offsets));
+    }
+    return Status::OK();
+}
+
+}  // namespace
+
 Result<std::vector<RealtimeCommitProgress>> FileStoreWrite::PrepareCommitWithProgress(int64_t) {
     return Status::Invalid("prepare commit with progress requires a real-time writer");
 }
@@ -144,17 +165,8 @@ Result<std::unique_ptr<FileStoreWrite>> FileStoreWrite::Create(std::unique_ptr<W
             if (options.DeletionVectorsEnabled()) {
                 return Status::Invalid("real-time append write does not support deletion vectors");
             }
-            PAIMON_ASSIGN_OR_RAISE(std::optional<Snapshot> latest_snapshot,
-                                   snapshot_manager->LatestSnapshot());
-            if (latest_snapshot) {
-                PAIMON_ASSIGN_OR_RAISE(RealtimeOffsetMap realtime_committed_offsets,
-                                       RealtimeCommitProperties::ReadOffsets(
-                                           latest_snapshot, options.GetFileSystem()));
-                PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
-                                       RealtimeContextImpl::Cast(ctx->GetRealtimeContext()));
-                PAIMON_RETURN_NOT_OK(realtime_context_impl->AdvanceCommittedProgress(
-                    latest_snapshot->Id(), realtime_committed_offsets));
-            }
+            PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx->GetRealtimeContext(),
+                                                                  snapshot_manager, options));
         }
         std::shared_ptr<arrow::Schema> write_schema = arrow_schema;
         const auto& write_field_names = ctx->GetWriteSchema();
@@ -207,17 +219,8 @@ Result<std::unique_ptr<FileStoreWrite>> FileStoreWrite::Create(std::unique_ptr<W
                 return Status::NotImplemented(
                     "PK realtime v1 does not support a custom write schema");
             }
-            PAIMON_ASSIGN_OR_RAISE(std::optional<Snapshot> latest_snapshot,
-                                   snapshot_manager->LatestSnapshot());
-            if (latest_snapshot) {
-                PAIMON_ASSIGN_OR_RAISE(RealtimeOffsetMap realtime_committed_offsets,
-                                       RealtimeCommitProperties::ReadOffsets(
-                                           latest_snapshot, options.GetFileSystem()));
-                PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
-                                       RealtimeContextImpl::Cast(ctx->GetRealtimeContext()));
-                PAIMON_RETURN_NOT_OK(realtime_context_impl->AdvanceCommittedProgress(
-                    latest_snapshot->Id(), realtime_committed_offsets));
-            }
+            PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx->GetRealtimeContext(),
+                                                                  snapshot_manager, options));
         }
         if (options.GetBucket() == BucketModeDefine::POSTPONE_BUCKET) {
             return PostponeBucketFileStoreWrite::Create(
diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp
index d8e7f5d15..c6a62c107 100644
--- a/src/paimon/core/operation/key_value_file_store_write.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write.cpp
@@ -154,7 +154,8 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
             RealtimeStoreState store_state,
             realtime_context_impl->GetOrCreateRealtimeStore(
                 RealtimeStoreCreateRequest{std::move(c_write_schema), options_.ToMap(), pool_,
-                                           partition_map, bucket, RealtimeStoreMode::PRIMARY_KEY}));
+                                           RealtimeStoreMode::PRIMARY_KEY},
+                RealtimePartitionBucket(partition_map, bucket)));
         realtime_store_state = std::move(store_state);
         compact_manager = std::make_shared<NoopCompactManager>();
     } else {
diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp
index c85e75ee0..02a7fac20 100644
--- a/src/paimon/core/operation/merge_file_split_read.cpp
+++ b/src/paimon/core/operation/merge_file_split_read.cpp
@@ -81,8 +81,9 @@ class MergeFunctionWrapper;
 class MergeFileSplitRead::RealtimeReaderBuilder {
  public:
     static Result<std::unique_ptr<BatchReader>> Create(
-        MergeFileSplitRead* owner, const std::vector<std::shared_ptr<Split>>& disk_splits,
-        std::vector<std::unique_ptr<KeyValueRecordReader>>&& additional_readers) {
+        const std::vector<std::shared_ptr<Split>>& disk_splits,
+        std::vector<std::unique_ptr<KeyValueRecordReader>>&& additional_readers,
+        MergeFileSplitRead* owner) {
         RealtimeReaderBuilder builder(owner);
         std::vector<std::unique_ptr<KeyValueRecordReader>> readers;
         if (!disk_splits.empty()) {
@@ -140,18 +141,18 @@ class MergeFileSplitRead::RealtimeReaderBuilder {
             }
         }
 
-        DeletionVector::Factory dv_factory = DeletionVector::CreateFactory(
-            owner_->options_.GetFileSystem(),
-            DeletionVector::CreateDeletionFileMap(data_files, deletion_files), owner_->pool_);
-        std::vector<std::vector<SortedRun>> disk_sections =
-            IntervalPartition(data_files, owner_->key_comparator_).Partition();
+        DeletionVector::Factory dv_factory;
+        std::vector<std::vector<SortedRun>> disk_sections;
+        PAIMON_RETURN_NOT_OK(
+            owner_->CreateDiskSections(data_files, deletion_files, &dv_factory, &disk_sections));
         for (const std::vector<SortedRun>& section : disk_sections) {
-            for (const SortedRun& run : section) {
-                PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> disk_reader,
-                                       owner_->CreateReaderForRun(partition, run, dv_factory,
-                                                                  owner_->predicate_for_keys_,
-                                                                  data_file_path_factory));
-                readers->push_back(std::move(disk_reader));
+            PAIMON_ASSIGN_OR_RAISE(
+                std::vector<std::unique_ptr<KeyValueRecordReader>> section_readers,
+                owner_->CreateRecordReadersForSection(section, partition, dv_factory,
+                                                      owner_->predicate_for_keys_,
+                                                      data_file_path_factory));
+            for (std::unique_ptr<KeyValueRecordReader>& reader : section_readers) {
+                readers->push_back(std::move(reader));
             }
         }
         return Status::OK();
@@ -165,34 +166,9 @@ class MergeFileSplitRead::RealtimeReaderBuilder {
         }
         PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<SortMergeReader> sort_merge_reader,
                                owner_->CreateSortMergeReader(std::move(record_readers)));
-        return CreateProjectedReader(std::move(sort_merge_reader));
-    }
-
-    Result<std::unique_ptr<BatchReader>> CreateProjectedReader(
-        std::unique_ptr<SortMergeReader>&& sort_merge_reader) {
-        if (!owner_->force_keep_delete_) {
-            sort_merge_reader = std::make_unique<DropDeleteReader>(std::move(sort_merge_reader));
-        }
-
-        std::unique_ptr<BatchReader> projection_reader;
-        if (!owner_->context_->EnableMultiThreadRowToBatch()) {
-            PAIMON_ASSIGN_OR_RAISE(
-                projection_reader,
-                KeyValueProjectionReader::Create(
-                    std::move(sort_merge_reader), owner_->raw_read_schema_, owner_->projection_,
-                    owner_->options_.GetReadBatchSize(), owner_->pool_));
-        } else {
-            const int32_t thread_number = owner_->context_->GetRowToBatchThreadNumber();
-            assert(thread_number > 0);
-            projection_reader = std::make_unique<AsyncKeyValueProjectionReader>(
-                std::move(sort_merge_reader), owner_->raw_read_schema_, owner_->projection_,
-                owner_->options_.GetReadBatchSize(), thread_number, owner_->pool_);
-        }
-        PAIMON_ASSIGN_OR_RAISE(projection_reader,
-                               owner_->ApplyPredicateFilterIfNeeded(
-                                   std::move(projection_reader), owner_->context_->GetPredicate()));
-        return std::make_unique<CompleteRowKindBatchReader>(std::move(projection_reader),
-                                                            owner_->pool_);
+        return owner_->CreateProjectedReader(std::move(sort_merge_reader),
+                                             owner_->context_->GetPredicate(),
+                                             /*complete_row_kind=*/true);
     }
 
     MergeFileSplitRead* owner_;
@@ -281,7 +257,7 @@ Result<std::unique_ptr<BatchReader>> MergeFileSplitRead::CreateReader(
 Result<std::unique_ptr<BatchReader>> MergeFileSplitRead::CreateRealtimeReader(
     const std::vector<std::shared_ptr<Split>>& disk_splits,
     std::vector<std::unique_ptr<KeyValueRecordReader>>&& additional_readers) {
-    return RealtimeReaderBuilder::Create(this, disk_splits, std::move(additional_readers));
+    return RealtimeReaderBuilder::Create(disk_splits, std::move(additional_readers), this);
 }
 
 void MergeFileSplitRead::SetMergeFunctionWrapper(
@@ -362,13 +338,10 @@ Result<std::unique_ptr<FileBatchReader>> MergeFileSplitRead::ApplyIndexAndDvRead
 Result<std::unique_ptr<BatchReader>> MergeFileSplitRead::CreateMergeReader(
     const std::shared_ptr<DataSplitImpl>& data_split,
     const std::shared_ptr<DataFilePathFactory>& data_file_path_factory) {
-    auto dv_factory = DeletionVector::CreateFactory(
-        options_.GetFileSystem(),
-        DeletionVector::CreateDeletionFileMap(data_split->DataFiles(), data_split->DeletionFiles()),
-        pool_);
-
-    std::vector<std::vector<SortedRun>> sections =
-        IntervalPartition(data_split->DataFiles(), key_comparator_).Partition();
+    DeletionVector::Factory dv_factory;
+    std::vector<std::vector<SortedRun>> sections;
+    PAIMON_RETURN_NOT_OK(CreateDiskSections(data_split->DataFiles(), data_split->DeletionFiles(),
+                                            &dv_factory, &sections));
     std::vector<std::unique_ptr<BatchReader>> batch_readers;
     batch_readers.reserve(sections.size());
     // no overlap through multiple sections
@@ -579,36 +552,75 @@ Result<std::unique_ptr<BatchReader>> MergeFileSplitRead::CreateReaderForSection(
     } else {
         predicate = context_->GetPredicate();
     }
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<SortMergeReader> sort_merge_reader,
-                           CreateSortMergeReaderForSection(section, partition, dv_factory,
-                                                           predicate, data_file_path_factory,
-                                                           /*drop_delete=*/!force_keep_delete_));
-    // KeyValueProjectionReader converts KeyValue objects to arrow array according to projection
-    if (!context_->EnableMultiThreadRowToBatch()) {
-        return KeyValueProjectionReader::Create(std::move(sort_merge_reader), raw_read_schema_,
-                                                projection_, options_.GetReadBatchSize(), pool_);
-    }
-    int32_t thread_number = context_->GetRowToBatchThreadNumber();
-    assert(thread_number > 0);
-    return std::make_unique<AsyncKeyValueProjectionReader>(
-        std::move(sort_merge_reader), raw_read_schema_, projection_, options_.GetReadBatchSize(),
-        thread_number, pool_);
+    PAIMON_ASSIGN_OR_RAISE(
+        std::unique_ptr<SortMergeReader> sort_merge_reader,
+        CreateSortMergeReaderForSection(section, partition, dv_factory, predicate,
+                                        data_file_path_factory, /*drop_delete=*/false));
+    return CreateProjectedReader(std::move(sort_merge_reader), /*predicate=*/nullptr,
+                                 /*complete_row_kind=*/false);
 }
 
-Result<std::unique_ptr<SortMergeReader>> MergeFileSplitRead::CreateSortMergeReaderForSection(
+Status MergeFileSplitRead::CreateDiskSections(
+    const std::vector<std::shared_ptr<DataFileMeta>>& data_files,
+    const std::vector<std::optional<DeletionFile>>& deletion_files,
+    DeletionVector::Factory* dv_factory, std::vector<std::vector<SortedRun>>* sections) const {
+    *dv_factory = DeletionVector::CreateFactory(
+        options_.GetFileSystem(), DeletionVector::CreateDeletionFileMap(data_files, deletion_files),
+        pool_);
+    *sections = IntervalPartition(data_files, key_comparator_).Partition();
+    return Status::OK();
+}
+
+Result<std::vector<std::unique_ptr<KeyValueRecordReader>>>
+MergeFileSplitRead::CreateRecordReadersForSection(
     const std::vector<SortedRun>& section, const BinaryRow& partition,
     DeletionVector::Factory dv_factory, const std::shared_ptr<Predicate>& predicate,
-    const std::shared_ptr<DataFilePathFactory>& data_file_path_factory, bool drop_delete) {
-    // with overlap in one section
+    const std::shared_ptr<DataFilePathFactory>& data_file_path_factory) const {
     std::vector<std::unique_ptr<KeyValueRecordReader>> record_readers;
     record_readers.reserve(section.size());
-    for (const auto& run : section) {
-        // no overlap in a run
+    for (const SortedRun& run : section) {
         PAIMON_ASSIGN_OR_RAISE(
             std::unique_ptr<KeyValueRecordReader> run_reader,
             CreateReaderForRun(partition, run, dv_factory, predicate, data_file_path_factory));
         record_readers.emplace_back(std::move(run_reader));
     }
+    return record_readers;
+}
+
+Result<std::unique_ptr<BatchReader>> MergeFileSplitRead::CreateProjectedReader(
+    std::unique_ptr<SortMergeReader>&& sort_merge_reader,
+    const std::shared_ptr<Predicate>& predicate, bool complete_row_kind) {
+    if (!force_keep_delete_) {
+        sort_merge_reader = std::make_unique<DropDeleteReader>(std::move(sort_merge_reader));
+    }
+    std::unique_ptr<BatchReader> projection_reader;
+    if (!context_->EnableMultiThreadRowToBatch()) {
+        PAIMON_ASSIGN_OR_RAISE(
+            projection_reader,
+            KeyValueProjectionReader::Create(std::move(sort_merge_reader), raw_read_schema_,
+                                             projection_, options_.GetReadBatchSize(), pool_));
+    } else {
+        const int32_t thread_number = context_->GetRowToBatchThreadNumber();
+        assert(thread_number > 0);
+        projection_reader = std::make_unique<AsyncKeyValueProjectionReader>(
+            std::move(sort_merge_reader), raw_read_schema_, projection_,
+            options_.GetReadBatchSize(), thread_number, pool_);
+    }
+    PAIMON_ASSIGN_OR_RAISE(projection_reader,
+                           ApplyPredicateFilterIfNeeded(std::move(projection_reader), predicate));
+    if (complete_row_kind) {
+        return std::make_unique<CompleteRowKindBatchReader>(std::move(projection_reader), pool_);
+    }
+    return projection_reader;
+}
+
+Result<std::unique_ptr<SortMergeReader>> MergeFileSplitRead::CreateSortMergeReaderForSection(
+    const std::vector<SortedRun>& section, const BinaryRow& partition,
+    DeletionVector::Factory dv_factory, const std::shared_ptr<Predicate>& predicate,
+    const std::shared_ptr<DataFilePathFactory>& data_file_path_factory, bool drop_delete) {
+    PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> record_readers,
+                           CreateRecordReadersForSection(section, partition, dv_factory, predicate,
+                                                         data_file_path_factory));
     PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<SortMergeReader> sort_merge_reader,
                            CreateSortMergeReader(std::move(record_readers)));
     if (drop_delete) {
diff --git a/src/paimon/core/operation/merge_file_split_read.h b/src/paimon/core/operation/merge_file_split_read.h
index f01b252be..ad45e4cf5 100644
--- a/src/paimon/core/operation/merge_file_split_read.h
+++ b/src/paimon/core/operation/merge_file_split_read.h
@@ -143,6 +143,20 @@ class MergeFileSplitRead : public AbstractSplitRead {
         DeletionVector::Factory dv_factory,
         const std::shared_ptr<DataFilePathFactory>& data_file_path_factory);
 
+    Status CreateDiskSections(const std::vector<std::shared_ptr<DataFileMeta>>& data_files,
+                              const std::vector<std::optional<DeletionFile>>& deletion_files,
+                              DeletionVector::Factory* dv_factory,
+                              std::vector<std::vector<SortedRun>>* sections) const;
+
+    Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateRecordReadersForSection(
+        const std::vector<SortedRun>& section, const BinaryRow& partition,
+        DeletionVector::Factory dv_factory, const std::shared_ptr<Predicate>& predicate,
+        const std::shared_ptr<DataFilePathFactory>& data_file_path_factory) const;
+
+    Result<std::unique_ptr<BatchReader>> CreateProjectedReader(
+        std::unique_ptr<SortMergeReader>&& sort_merge_reader,
+        const std::shared_ptr<Predicate>& predicate, bool complete_row_kind);
+
     Result<std::unique_ptr<KeyValueRecordReader>> CreateReaderForRun(
         const BinaryRow& partition, const SortedRun& sorted_run, DeletionVector::Factory dv_factory,
         const std::shared_ptr<Predicate>& predicate,
diff --git a/src/paimon/core/realtime/arrow_realtime_store_test.cpp b/src/paimon/core/realtime/arrow_realtime_store_test.cpp
index 0a3e52353..d4bda2000 100644
--- a/src/paimon/core/realtime/arrow_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/arrow_realtime_store_test.cpp
@@ -233,11 +233,7 @@ TEST_F(ArrowRealtimeStoreTest, TestFullStatisticsPrunesNonMatchingBatch) {
     ArrowRealtimeStoreFactory factory;
     std::unique_ptr<ArrowSchema> write_schema = MakeReadSchema(schema_);
     RealtimeStoreCreateRequest request{std::move(write_schema),
-                                       /*options=*/{},
-                                       pool_,
-                                       /*partition=*/{},
-                                       /*bucket=*/0,
-                                       RealtimeStoreMode::APPEND_ONLY,
+                                       /*options=*/{}, pool_, RealtimeStoreMode::APPEND_ONLY,
                                        StatisticsMode::FULL};
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeStore> realtime_store,
                          factory.Create(std::move(request)));
@@ -279,11 +275,7 @@ TEST_F(ArrowRealtimeStoreTest, TestFactoryRejectsInvalidMode) {
     ArrowRealtimeStoreFactory factory;
     std::unique_ptr<ArrowSchema> write_schema = MakeReadSchema(schema_);
     RealtimeStoreCreateRequest request{std::move(write_schema),
-                                       /*options=*/{},
-                                       pool_,
-                                       /*partition=*/{},
-                                       /*bucket=*/0,
-                                       static_cast<RealtimeStoreMode>(-1)};
+                                       /*options=*/{}, pool_, static_cast<RealtimeStoreMode>(-1)};
     ASSERT_NOK_WITH_MSG(factory.Create(std::move(request)), "invalid real-time store mode: -1");
 }
 
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index 53652cc81..d8a4cfdc0 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -292,8 +292,8 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
             }
             std::shared_ptr<arrow::StructArray> data_batch =
                 checked_pointer_cast<arrow::StructArray>(arrow_array);
-            Status transport_status =
-                ValidatePreparedTransportSchema(arrow::schema(data_batch->type()->fields()));
+            Status transport_status = PreparedKeyValueReaderFactory::ValidateTransportSchema(
+                arrow::schema(data_batch->type()->fields()));
             if (!transport_status.ok()) {
                 return Status::Invalid(
                     "prepared batch field does not match prepared transport "
@@ -420,7 +420,8 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
 
 }  // namespace
 
-Status ValidatePreparedTransportSchema(const std::shared_ptr<arrow::Schema>& prepared_schema) {
+Status PreparedKeyValueReaderFactory::ValidateTransportSchema(
+    const std::shared_ptr<arrow::Schema>& prepared_schema) {
     if (!prepared_schema || prepared_schema->num_fields() < kPreparedValueStartIndex) {
         return Status::Invalid("prepared schema must contain realtime transport fields");
     }
@@ -450,7 +451,7 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderImpl(
     if (visible_offsets.has_value() && visible_offsets->begin > visible_offsets->end) {
         return Status::Invalid("prepared visible offset range begin exceeds end");
     }
-    PAIMON_RETURN_NOT_OK(ValidatePreparedTransportSchema(prepared_schema));
+    PAIMON_RETURN_NOT_OK(PreparedKeyValueReaderFactory::ValidateTransportSchema(prepared_schema));
     if (!key_schema) {
         return Status::Invalid("prepared key schema cannot be null");
     }
@@ -476,7 +477,7 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderImpl(
 
 }  // namespace
 
-Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
+Result<std::unique_ptr<KeyValueRecordReader>> PreparedKeyValueReaderFactory::Create(
     std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
     const std::optional<OffsetRange>& visible_offsets,
     const std::shared_ptr<arrow::Schema>& key_schema,
@@ -487,7 +488,8 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
                                         /*offset_coverage=*/nullptr);
 }
 
-Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> AdaptPreparedCommitBatchReaders(
+Result<std::vector<std::unique_ptr<KeyValueRecordReader>>>
+PreparedKeyValueReaderFactory::CreateForCommit(
     std::vector<std::unique_ptr<BatchReader>>&& readers,
     const std::shared_ptr<arrow::Schema>& prepared_schema, const OffsetRange& sealed_offsets,
     const std::shared_ptr<arrow::Schema>& key_schema,
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.h b/src/paimon/core/realtime/prepared_key_value_reader.h
index 4ef4887e6..389c75f9a 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.h
+++ b/src/paimon/core/realtime/prepared_key_value_reader.h
@@ -32,23 +32,27 @@ namespace paimon {
 class BatchReader;
 class MemoryPool;
 
-/// Validates the required leading fields of a prepared real-time transport schema.
-Status ValidatePreparedTransportSchema(const std::shared_ptr<arrow::Schema>& prepared_schema);
-
-/// Adapts a plugin query reader and limits its rows to `visible_offsets` when present.
-Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
-    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
-    const std::optional<OffsetRange>& visible_offsets,
-    const std::shared_ptr<arrow::Schema>& key_schema,
-    const std::shared_ptr<arrow::Schema>& value_schema,
-    const std::shared_ptr<MemoryPool>& memory_pool);
-
-/// Adapts commit readers and validates their offsets against `sealed_offsets`.
-Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> AdaptPreparedCommitBatchReaders(
-    std::vector<std::unique_ptr<BatchReader>>&& readers,
-    const std::shared_ptr<arrow::Schema>& prepared_schema, const OffsetRange& sealed_offsets,
-    const std::shared_ptr<arrow::Schema>& key_schema,
-    const std::shared_ptr<arrow::Schema>& value_schema,
-    const std::shared_ptr<MemoryPool>& memory_pool);
+class PreparedKeyValueReaderFactory {
+ public:
+    PreparedKeyValueReaderFactory() = delete;
+    ~PreparedKeyValueReaderFactory() = delete;
+
+    static Status ValidateTransportSchema(const std::shared_ptr<arrow::Schema>& prepared_schema);
+
+    static Result<std::unique_ptr<KeyValueRecordReader>> Create(
+        std::unique_ptr<BatchReader>&& reader,
+        const std::shared_ptr<arrow::Schema>& prepared_schema,
+        const std::optional<OffsetRange>& visible_offsets,
+        const std::shared_ptr<arrow::Schema>& key_schema,
+        const std::shared_ptr<arrow::Schema>& value_schema,
+        const std::shared_ptr<MemoryPool>& memory_pool);
+
+    static Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateForCommit(
+        std::vector<std::unique_ptr<BatchReader>>&& readers,
+        const std::shared_ptr<arrow::Schema>& prepared_schema, const OffsetRange& sealed_offsets,
+        const std::shared_ptr<arrow::Schema>& key_schema,
+        const std::shared_ptr<arrow::Schema>& value_schema,
+        const std::shared_ptr<MemoryPool>& memory_pool);
+};
 
 }  // namespace paimon
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index cfdf88f49..75aa15271 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -362,7 +362,7 @@ class PrimaryKeyRealtimeStore::Impl {
         }
         PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> read_schema,
                                           arrow::ImportSchema(context.read_schema));
-        PAIMON_RETURN_NOT_OK(ValidatePreparedTransportSchema(read_schema));
+        PAIMON_RETURN_NOT_OK(PreparedKeyValueReaderFactory::ValidateTransportSchema(read_schema));
         std::vector<std::unique_ptr<BatchReader>> readers;
         for (const std::shared_ptr<Segment>& segment : typed->Segments()) {
             for (const StoredBatch& batch : segment->Batches()) {
@@ -411,7 +411,7 @@ PrimaryKeyRealtimeStore::~PrimaryKeyRealtimeStore() = default;
 
 Result<std::shared_ptr<PrimaryKeyRealtimeStore>> PrimaryKeyRealtimeStore::Create(
     const std::shared_ptr<arrow::Schema>& prepared_schema) {
-    PAIMON_RETURN_NOT_OK(ValidatePreparedTransportSchema(prepared_schema));
+    PAIMON_RETURN_NOT_OK(PreparedKeyValueReaderFactory::ValidateTransportSchema(prepared_schema));
     return std::shared_ptr<PrimaryKeyRealtimeStore>(
         new PrimaryKeyRealtimeStore(std::make_unique<Impl>(prepared_schema)));
 }
diff --git a/src/paimon/core/realtime/realtime_append_only_writer.cpp b/src/paimon/core/realtime/realtime_append_only_writer.cpp
index 0fb4e58e7..632d64e16 100644
--- a/src/paimon/core/realtime/realtime_append_only_writer.cpp
+++ b/src/paimon/core/realtime/realtime_append_only_writer.cpp
@@ -55,11 +55,11 @@ Result<std::shared_ptr<RealtimeAppendOnlyWriter>> RealtimeAppendOnlyWriter::Crea
     }
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
                            RealtimeContextImpl::Cast(realtime_context));
-    RealtimeStoreCreateRequest request{
-        std::move(write_schema),        options,        memory_pool, partition, bucket,
-        RealtimeStoreMode::APPEND_ONLY, statistics_mode};
+    RealtimeStoreCreateRequest request{std::move(write_schema), options, memory_pool,
+                                       RealtimeStoreMode::APPEND_ONLY, statistics_mode};
     PAIMON_ASSIGN_OR_RAISE(RealtimeStoreState store_state,
-                           realtime_context_impl->GetOrCreateRealtimeStore(std::move(request)));
+                           realtime_context_impl->GetOrCreateRealtimeStore(
+                               std::move(request), RealtimePartitionBucket(partition, bucket)));
     return std::shared_ptr<RealtimeAppendOnlyWriter>(new RealtimeAppendOnlyWriter(
         store_state.store, file_writer, input_schema, store_state.initial_offset, memory_pool));
 }
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index 052b616f6..4b5da718e 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -96,7 +96,7 @@ Status RealtimeContextImpl::Start() {
 }
 
 Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
-    RealtimeStoreCreateRequest&& request) {
+    RealtimeStoreCreateRequest&& request, const RealtimePartitionBucket& partition_bucket) {
     if (!request.write_schema || !request.write_schema->release) {
         return Status::Invalid("real-time store write schema is null");
     }
@@ -107,10 +107,9 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     schema_guard.Release();
     std::lock_guard<std::mutex> progress_lock(progress_mutex_);
     std::lock_guard<std::mutex> registry_lock(mutex_);
-    const RealtimePartitionBucket key(request.partition, request.bucket);
-    auto iter = stores_.find(key);
+    auto iter = stores_.find(partition_bucket);
     int64_t initial_offset = 0;
-    auto offset_iter = committed_offsets_.find(key);
+    auto offset_iter = committed_offsets_.find(partition_bucket);
     if (offset_iter != committed_offsets_.end()) {
         if (offset_iter->second == std::numeric_limits<int64_t>::max()) {
             return Status::Invalid("real-time offset has reached INT64_MAX");
@@ -121,8 +120,9 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
         if (iter->second.mode != request.mode ||
             !iter->second.write_schema->Equals(*requested_schema, /*check_metadata=*/true)) {
             return Status::Invalid("real-time store schema or mode mismatch for partition " +
-                                   PartitionToString(key.partition) + ", bucket " +
-                                   std::to_string(key.bucket) + "; recreate the RealtimeContext");
+                                   PartitionToString(partition_bucket.partition) + ", bucket " +
+                                   std::to_string(partition_bucket.bucket) +
+                                   "; recreate the RealtimeContext");
         }
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeReadView> read_view,
                                iter->second.store->AcquireReadView());
@@ -150,9 +150,9 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     RealtimeStoreMode mode = request.mode;
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> store,
                            factory_->Create(std::move(request)));
-    stores_.emplace(key, StoreEntry{store, requested_schema, mode});
+    stores_.emplace(partition_bucket, StoreEntry{store, requested_schema, mode});
     if (offset_iter != committed_offsets_.end()) {
-        reclaimed_offsets_.emplace(key, offset_iter->second);
+        reclaimed_offsets_.emplace(partition_bucket, offset_iter->second);
     }
     return RealtimeStoreState{std::move(store), initial_offset};
 }
diff --git a/src/paimon/core/realtime/realtime_context_impl.h b/src/paimon/core/realtime/realtime_context_impl.h
index 29ac7c05d..ea069a5cd 100644
--- a/src/paimon/core/realtime/realtime_context_impl.h
+++ b/src/paimon/core/realtime/realtime_context_impl.h
@@ -69,7 +69,8 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
     static Result<std::shared_ptr<RealtimeContextImpl>> Cast(
         const std::shared_ptr<RealtimeContext>& context);
 
-    Result<RealtimeStoreState> GetOrCreateRealtimeStore(RealtimeStoreCreateRequest&& request);
+    Result<RealtimeStoreState> GetOrCreateRealtimeStore(
+        RealtimeStoreCreateRequest&& request, const RealtimePartitionBucket& partition_bucket);
 
     Result<int64_t> AdvanceMaterializedMaxSequenceNumber(
         const RealtimePartitionBucket& partition_bucket, int64_t max_sequence_number);
diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp
index fbdaa86b7..a418c17a8 100644
--- a/src/paimon/core/realtime/realtime_context_test.cpp
+++ b/src/paimon/core/realtime/realtime_context_test.cpp
@@ -120,8 +120,9 @@ Result<RealtimeStoreState> GetOrCreateAppendStore(
     const std::shared_ptr<MemoryPool>& memory_pool,
     StatisticsMode statistics_mode = StatisticsMode::NONE) {
     return context->GetOrCreateRealtimeStore(
-        RealtimeStoreCreateRequest{std::move(write_schema), options, memory_pool, partition, bucket,
-                                   RealtimeStoreMode::APPEND_ONLY, statistics_mode});
+        RealtimeStoreCreateRequest{std::move(write_schema), options, memory_pool,
+                                   RealtimeStoreMode::APPEND_ONLY, statistics_mode},
+        RealtimePartitionBucket(partition, bucket));
 }
 
 TEST(RealtimeContextTest, TestReusesStoreAndCapturesRegisteredViews) {
@@ -170,8 +171,10 @@ TEST(RealtimeContextTest, TestRejectsMismatchedModeOnStoreReuse) {
         GetOrCreateAppendStore(context, partition, 0, MakeWriteSchema(), {}, GetDefaultPool()));
 
     ASSERT_NOK_WITH_MSG(
-        context->GetOrCreateRealtimeStore(RealtimeStoreCreateRequest{
-            MakeWriteSchema(), {}, GetDefaultPool(), partition, 0, RealtimeStoreMode::PRIMARY_KEY}),
+        context->GetOrCreateRealtimeStore(
+            RealtimeStoreCreateRequest{
+                MakeWriteSchema(), {}, GetDefaultPool(), RealtimeStoreMode::PRIMARY_KEY},
+            RealtimePartitionBucket(partition, 0)),
         "schema or mode mismatch for partition {dt=2026-08-02}, bucket 0; recreate the "
         "RealtimeContext");
     ASSERT_EQ(1, factory->stores.size());
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index bcc8f6705..692eff1bd 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -283,10 +283,10 @@ Status RealtimePrimaryKeyWriter::FlushSegment(const std::shared_ptr<RealtimeSegm
                                               const OffsetRange& sealed_offsets) {
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
                            realtime_store_->CreateCommitReaders(segment));
-    PAIMON_ASSIGN_OR_RAISE(
-        std::vector<std::unique_ptr<KeyValueRecordReader>> prepared_readers,
-        AdaptPreparedCommitBatchReaders(std::move(readers), prepared_schema_, sealed_offsets,
-                                        key_schema_, write_schema_, memory_pool_));
+    PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> prepared_readers,
+                           PreparedKeyValueReaderFactory::CreateForCommit(
+                               std::move(readers), prepared_schema_, sealed_offsets, key_schema_,
+                               write_schema_, memory_pool_));
     std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
     sorted_readers.reserve(prepared_readers.size());
     for (std::unique_ptr<KeyValueRecordReader>& prepared_reader : prepared_readers) {
@@ -295,7 +295,7 @@ Status RealtimePrimaryKeyWriter::FlushSegment(const std::shared_ptr<RealtimeSegm
             std::move(prepared_reader), key_comparator_,
             std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge_function))));
     }
-    return merge_tree_writer_->WriteSortedReaders(std::move(sorted_readers));
+    return merge_tree_writer_->WriteSortedReadersToFiles(std::move(sorted_readers));
 }
 
 Status RealtimePrimaryKeyWriter::Compact(bool) {
diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp
index 050f09701..cc1e6a076 100644
--- a/src/paimon/core/schema/schema_validation_test.cpp
+++ b/src/paimon/core/schema/schema_validation_test.cpp
@@ -46,11 +46,12 @@ TEST(SchemaValidationTest, TestSimple) {
     ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema));
 }
 
-TEST(SchemaValidationTest, TestRealtimeOffsetIsNotGloballyReserved) {
+TEST(SchemaValidationTest, TestRealtimeOffsetIsGloballyReserved) {
     auto schema = arrow::schema({arrow::field("_REALTIME_OFFSET", arrow::int64())});
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<TableSchema> table_schema,
                          TableSchema::Create(0, schema, {}, {}, {}));
-    ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema));
+    ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
+                        "field name '_REALTIME_OFFSET' in schema cannot be special field");
 }
 
 TEST(SchemaValidationTest, TestVectorType) {
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index 64b866439..e13140f54 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -32,7 +32,7 @@
 #include "paimon/core/global_index/indexed_split_impl.h"
 #include "paimon/core/io/merged_key_value_record_reader.h"
 #include "paimon/core/key_value.h"
-#include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
+#include "paimon/core/mergetree/compact/merge_function.h"
 #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
 #include "paimon/core/operation/merge_file_split_read.h"
 #include "paimon/core/operation/raw_file_split_read.h"
@@ -42,6 +42,7 @@
 #include "paimon/core/table/source/data_split_impl.h"
 #include "paimon/core/table/source/pk_count_reader.h"
 #include "paimon/core/table/source/realtime_split.h"
+#include "paimon/core/utils/primary_key_table_utils.h"
 #include "paimon/status.h"
 
 namespace paimon {
@@ -59,6 +60,7 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateMemoryReaders(
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
     const std::shared_ptr<FieldsComparator>& key_comparator,
+    const std::shared_ptr<InternalReadContext>& context,
     const std::shared_ptr<MemoryPool>& memory_pool) {
     arrow::FieldVector prepared_fields = {
         DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
@@ -87,12 +89,16 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateMemoryReaders(
         if (!reader) {
             return Status::Invalid("PK real-time store returned a null query reader");
         }
-        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> prepared_reader,
-                               AdaptPreparedBatchReader(std::move(reader), prepared_schema,
-                                                        OffsetRange(split->CommittedEndOffset(),
-                                                                    split->MemoryEndOffset()),
-                                                        key_schema, value_schema, memory_pool));
-        auto merge = std::make_unique<DeduplicateMergeFunction>(false);
+        PAIMON_ASSIGN_OR_RAISE(
+            std::unique_ptr<KeyValueRecordReader> prepared_reader,
+            PreparedKeyValueReaderFactory::Create(
+                std::move(reader), prepared_schema,
+                OffsetRange(split->CommittedEndOffset(), split->MemoryEndOffset()), key_schema,
+                value_schema, memory_pool));
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<MergeFunction> merge,
+                               PrimaryKeyTableUtils::CreateMergeFunction(
+                                   value_schema, context->GetTableSchema()->PrimaryKeys(),
+                                   context->GetCoreOptions(), memory_pool));
         result.push_back(std::make_unique<MergedKeyValueRecordReader>(
             std::move(prepared_reader), key_comparator,
             std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge))));
@@ -273,7 +279,7 @@ Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateRealtimeReader(
                 std::vector<std::unique_ptr<KeyValueRecordReader>> memory_readers,
                 CreateMemoryReaders(realtime_split, memory, merge_read->GetKeySchema(),
                                     merge_read->GetValueSchema(), merge_read->GetKeyComparator(),
-                                    GetMemoryPool()));
+                                    context_, GetMemoryPool()));
             PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> reader,
                                    merge_read->CreateRealtimeReader(realtime_split->DiskSplits(),
                                                                     std::move(memory_readers)));
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index 78e8badee..94bf81edc 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -52,6 +52,8 @@
 #include "paimon/core/core_options.h"
 #include "paimon/core/operation/commit/realtime_commit_properties.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
+#include "paimon/core/schema/schema_manager.h"
+#include "paimon/core/schema/table_schema.h"
 #include "paimon/core/table/sink/commit_message_impl.h"
 #include "paimon/core/table/source/realtime_split.h"
 #include "paimon/core/utils/snapshot_manager.h"
@@ -1200,11 +1202,24 @@ class RealtimeWriteInteTest : public ::testing::Test {
         if (views.size() != 1) {
             return Status::Invalid("expected one PK real-time read view");
         }
+        PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options_));
+        SchemaManager schema_manager(core_options.GetFileSystem(), table_path_);
+        PAIMON_ASSIGN_OR_RAISE(std::optional<std::shared_ptr<TableSchema>> table_schema,
+                               schema_manager.Latest());
+        if (!table_schema) {
+            return Status::Invalid("expected a table schema");
+        }
         auto read_schema = std::make_unique<ArrowSchema>();
         arrow::FieldVector requested_fields = {
-            DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())};
-        requested_fields.insert(requested_fields.end(), schema_->fields().begin(),
-                                schema_->fields().end());
+            DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())
+                ->WithNullable(false),
+            DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
+                ->WithNullable(false),
+            DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset())};
+        std::shared_ptr<arrow::Schema> value_schema =
+            DataField::ConvertDataFieldsToArrowSchema(table_schema.value()->Fields());
+        requested_fields.insert(requested_fields.end(), value_schema->fields().begin(),
+                                value_schema->fields().end());
         PAIMON_RETURN_NOT_OK_FROM_ARROW(
             arrow::ExportSchema(*arrow::schema(requested_fields), read_schema.get()));
         ScopeGuard schema_guard([schema = read_schema.get()]() { ArrowSchemaRelease(schema); });

From bf00d530445fcd52c82427ec25b0354887879b89 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Tue, 25 Aug 2026 22:55:52 +0800
Subject: [PATCH 35/62] fix(realtime): tighten primary key framework boundaries

---
 src/paimon/CMakeLists.txt                     |   2 +
 .../core/operation/file_store_write.cpp       |   4 +-
 .../key_value_file_store_write_test.cpp       |  24 ++-
 .../realtime/arrow_realtime_store_factory.cpp |   5 +-
 .../primary_key_realtime_validator.cpp        |  80 ++++++++++
 .../primary_key_realtime_validator.h          |  36 +++++
 .../primary_key_realtime_validator_test.cpp   |  91 ++++++++++++
 .../realtime/primary_key_realtime_store.cpp   |  76 +++-------
 .../realtime/primary_key_realtime_store.h     |   7 +-
 .../primary_key_realtime_store_test.cpp       | 139 ++++++++++--------
 .../realtime/realtime_primary_key_writer.cpp  |  23 ++-
 src/paimon/core/table/source/table_scan.cpp   |   5 +-
 12 files changed, 346 insertions(+), 146 deletions(-)
 create mode 100644 src/paimon/core/realtime/framework/primary_key_realtime_validator.cpp
 create mode 100644 src/paimon/core/realtime/framework/primary_key_realtime_validator.h
 create mode 100644 src/paimon/core/realtime/framework/primary_key_realtime_validator_test.cpp

diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index 0a78b0902..7ff15100b 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -382,6 +382,7 @@ set(PAIMON_CORE_SRCS
     core/operation/write_restore.cpp
     core/realtime/arrow_realtime_store.cpp
     core/realtime/arrow_realtime_store_factory.cpp
+    core/realtime/framework/primary_key_realtime_validator.cpp
     core/realtime/prepared_key_value_reader.cpp
     core/realtime/primary_key_realtime_store.cpp
     core/realtime/realtime_append_only_writer.cpp
@@ -790,6 +791,7 @@ if(PAIMON_BUILD_TESTS)
                     core/manifest/index_manifest_file_handler_test.cpp
                     core/memory/writer_memory_manager_test.cpp
                     core/realtime/arrow_realtime_store_test.cpp
+                    core/realtime/framework/primary_key_realtime_validator_test.cpp
                     core/realtime/primary_key_realtime_store_test.cpp
                     core/realtime/realtime_context_test.cpp
                     core/realtime/realtime_reader_test.cpp
diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp
index 26befb7ee..f77dadda2 100644
--- a/src/paimon/core/operation/file_store_write.cpp
+++ b/src/paimon/core/operation/file_store_write.cpp
@@ -36,7 +36,7 @@
 #include "paimon/core/operation/key_value_file_store_write.h"
 #include "paimon/core/options/merge_engine.h"
 #include "paimon/core/postpone/postpone_bucket_file_store_write.h"
-#include "paimon/core/realtime/primary_key_realtime_store.h"
+#include "paimon/core/realtime/framework/primary_key_realtime_validator.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
 #include "paimon/core/schema/schema_manager.h"
 #include "paimon/core/schema/table_schema.h"
@@ -210,7 +210,7 @@ Result<std::unique_ptr<FileStoreWrite>> FileStoreWrite::Create(std::unique_ptr<W
     } else {
         // pk table
         if (ctx->GetRealtimeContext()) {
-            PAIMON_RETURN_NOT_OK(PrimaryKeyRealtimeStore::ValidateOptions(options, *schema));
+            PAIMON_RETURN_NOT_OK(PrimaryKeyRealtimeValidator::ValidateOptions(options, *schema));
             if (ignore_previous_files) {
                 return Status::NotImplemented(
                     "PK realtime v1 requires restore from the latest snapshot");
diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp
index 0f70235a7..d52841f95 100644
--- a/src/paimon/core/operation/key_value_file_store_write_test.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp
@@ -488,19 +488,17 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeOffsetCollision) {
     });
     std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
     ASSERT_TRUE(dir);
-    CreateTable(dir->Str(), schema, options);
-
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create());
-    WriteContextBuilder builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), "test");
-    builder.SetOptions(options).WithStreamingMode(true).WithRealtimeContext(realtime_context);
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> write_context, builder.Finish());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                         FileStoreWrite::Create(std::move(write_context)));
-
-    ASSERT_NOK_WITH_MSG(writer->Write(MakeBatch(schema, R"([[1, 10]])")),
-                        "PK real-time write schema contains reserved transport field");
-    ASSERT_OK(writer->Close());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<Catalog> catalog, Catalog::Create(dir->Str(), options));
+    ASSERT_OK(catalog->CreateDatabase("foo", {}, /*ignore_if_exists=*/false));
+    ArrowSchema c_schema;
+    ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok());
+    Status create_status =
+        catalog->CreateTable(Identifier("foo", "bar"), &c_schema,
+                             /*partition_keys=*/{}, /*primary_keys=*/{"id"}, options,
+                             /*ignore_if_exists=*/false);
+    ArrowSchemaRelease(&c_schema);
+    ASSERT_NOK_WITH_MSG(create_status,
+                        "field name '_REALTIME_OFFSET' in schema cannot be special field");
 }
 
 TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) {
diff --git a/src/paimon/core/realtime/arrow_realtime_store_factory.cpp b/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
index d0d4ae704..dff12b589 100644
--- a/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
+++ b/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
@@ -49,8 +49,9 @@ Result<std::shared_ptr<RealtimeStore>> ArrowRealtimeStoreFactory::Create(
                                                         request.memory_pool, arrow_pool);
         }
         case RealtimeStoreMode::PRIMARY_KEY: {
-            PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                                   PrimaryKeyRealtimeStore::Create(imported_schema));
+            PAIMON_ASSIGN_OR_RAISE(
+                std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                PrimaryKeyRealtimeStore::Create(imported_schema, request.memory_pool));
             return std::shared_ptr<RealtimeStore>(std::move(store));
         }
     }
diff --git a/src/paimon/core/realtime/framework/primary_key_realtime_validator.cpp b/src/paimon/core/realtime/framework/primary_key_realtime_validator.cpp
new file mode 100644
index 000000000..61c640ee6
--- /dev/null
+++ b/src/paimon/core/realtime/framework/primary_key_realtime_validator.cpp
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/realtime/framework/primary_key_realtime_validator.h"
+
+#include <vector>
+
+#include "arrow/type.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/index/pk/primary_key_index_definitions.h"
+#include "paimon/core/schema/table_schema.h"
+#include "paimon/macros.h"
+
+namespace paimon {
+
+Status PrimaryKeyRealtimeValidator::ValidateOptions(const CoreOptions& options,
+                                                    const TableSchema& schema) {
+    if (options.GetBucket() <= 0) {
+        return Status::NotImplemented("PK realtime v1 requires fixed buckets");
+    }
+    if (options.GetMergeEngine() != MergeEngine::DEDUPLICATE) {
+        return Status::NotImplemented("PK realtime v1 supports only the DEDUPLICATE merge engine");
+    }
+    if (options.DataEvolutionEnabled()) {
+        return Status::NotImplemented("PK realtime v1 does not support data evolution");
+    }
+    if (!options.GetFieldsSequenceGroups().empty()) {
+        return Status::NotImplemented("PK realtime v1 does not support sequence groups");
+    }
+    if (options.IgnoreDelete() || options.PartialUpdateRemoveRecordOnDelete() ||
+        options.AggregationRemoveRecordOnDelete() ||
+        !options.GetPartialUpdateRemoveRecordOnSequenceGroup().empty()) {
+        return Status::NotImplemented("PK realtime v1 requires default delete behavior");
+    }
+    if (!options.GetSequenceField().empty()) {
+        return Status::NotImplemented("PK realtime v1 does not support sequence.field");
+    }
+    if (!options.SequenceFieldSortOrderIsAscending()) {
+        return Status::NotImplemented(
+            "PK realtime v1 supports only ascending sequence.field.sort-order");
+    }
+    if (options.NeedLookup() || options.DeletionVectorsEnabled() ||
+        options.GetChangelogProducer() != ChangelogProducer::NONE) {
+        return Status::NotImplemented("PK realtime v1 does not support lookup or early MOR");
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::vector<DataField> primary_key_fields,
+                           schema.TrimmedPrimaryKeyFields());
+    for (const DataField& field : primary_key_fields) {
+        if (field.Type()->id() == arrow::Type::FLOAT || field.Type()->id() == arrow::Type::DOUBLE) {
+            return Status::NotImplemented(
+                "PK realtime v1 does not support FLOAT or DOUBLE primary keys");
+        }
+    }
+    if (options.GlobalIndexEnabled()) {
+        PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions definitions,
+                               PrimaryKeyIndexDefinitions::Create(schema));
+        if (!definitions.Definitions().empty()) {
+            return Status::NotImplemented("PK realtime v1 does not support global indexes");
+        }
+    }
+    return Status::OK();
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/realtime/framework/primary_key_realtime_validator.h b/src/paimon/core/realtime/framework/primary_key_realtime_validator.h
new file mode 100644
index 000000000..fa9432d3b
--- /dev/null
+++ b/src/paimon/core/realtime/framework/primary_key_realtime_validator.h
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "paimon/status.h"
+
+namespace paimon {
+
+class CoreOptions;
+class TableSchema;
+
+class PrimaryKeyRealtimeValidator {
+ public:
+    PrimaryKeyRealtimeValidator() = delete;
+    ~PrimaryKeyRealtimeValidator() = delete;
+
+    static Status ValidateOptions(const CoreOptions& options, const TableSchema& schema);
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/realtime/framework/primary_key_realtime_validator_test.cpp b/src/paimon/core/realtime/framework/primary_key_realtime_validator_test.cpp
new file mode 100644
index 000000000..3b7793b14
--- /dev/null
+++ b/src/paimon/core/realtime/framework/primary_key_realtime_validator_test.cpp
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/realtime/framework/primary_key_realtime_validator.h"
+
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "arrow/type.h"
+#include "gtest/gtest.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/schema/table_schema.h"
+#include "paimon/defs.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+namespace {
+
+std::shared_ptr<TableSchema> PkSchema(
+    const std::shared_ptr<arrow::DataType>& key_type = arrow::int64(),
+    const std::map<std::string, std::string>& options = {}) {
+    return TableSchema::Create(
+               /*schema_id=*/0,
+               arrow::schema({arrow::field("id", key_type), arrow::field("value", arrow::utf8())}),
+               /*partition_keys=*/{}, /*primary_keys=*/{"id"}, options)
+        .value();
+}
+
+}  // namespace
+
+TEST(PrimaryKeyRealtimeValidatorTest, TestSupportedOptions) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
+    ASSERT_OK(PrimaryKeyRealtimeValidator::ValidateOptions(options, *PkSchema()));
+}
+
+TEST(PrimaryKeyRealtimeValidatorTest, TestUnsupportedOptions) {
+    const std::string sequence_group =
+        std::string(Options::FIELDS_PREFIX) + ".value." + Options::SEQUENCE_GROUP;
+    const std::vector<std::map<std::string, std::string>> unsupported_options = {
+        {{Options::BUCKET, "0"}},
+        {{Options::BUCKET, "1"}, {Options::MERGE_ENGINE, "partial-update"}},
+        {{Options::BUCKET, "1"}, {Options::DATA_EVOLUTION_ENABLED, "true"}},
+        {{Options::BUCKET, "1"}, {sequence_group, "seq"}},
+        {{Options::BUCKET, "1"}, {Options::SEQUENCE_FIELD, "seq"}},
+        {{Options::BUCKET, "1"}, {Options::FORCE_LOOKUP, "true"}},
+        {{Options::BUCKET, "1"}, {Options::DELETION_VECTORS_ENABLED, "true"}},
+        {{Options::BUCKET, "1"}, {Options::CHANGELOG_PRODUCER, "input"}},
+    };
+    for (const std::map<std::string, std::string>& option_map : unsupported_options) {
+        ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
+        ASSERT_NOK(PrimaryKeyRealtimeValidator::ValidateOptions(options, *PkSchema()));
+    }
+}
+
+TEST(PrimaryKeyRealtimeValidatorTest, TestRejectsFloatingPrimaryKeys) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
+    ASSERT_NOK_WITH_MSG(
+        PrimaryKeyRealtimeValidator::ValidateOptions(options, *PkSchema(arrow::float32())),
+        "FLOAT or DOUBLE primary keys");
+    ASSERT_NOK_WITH_MSG(
+        PrimaryKeyRealtimeValidator::ValidateOptions(options, *PkSchema(arrow::float64())),
+        "FLOAT or DOUBLE primary keys");
+}
+
+TEST(PrimaryKeyRealtimeValidatorTest, TestRejectsEnabledGlobalIndex) {
+    const std::map<std::string, std::string> option_map = {{Options::BUCKET, "1"},
+                                                           {Options::PK_BTREE_INDEX_COLUMNS, "id"}};
+    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
+    ASSERT_NOK_WITH_MSG(PrimaryKeyRealtimeValidator::ValidateOptions(
+                            options, *PkSchema(arrow::int64(), option_map)),
+                        "does not support global indexes");
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index 75aa15271..88e78680a 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -28,67 +28,17 @@
 #include "arrow/c/bridge.h"
 #include "fmt/format.h"
 #include "paimon/common/metrics/metrics_impl.h"
-#include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/arrow/arrow_utils.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
-#include "paimon/core/core_options.h"
-#include "paimon/core/index/pk/primary_key_index_definitions.h"
 #include "paimon/core/realtime/prepared_key_value_reader.h"
-#include "paimon/core/schema/table_schema.h"
 #include "paimon/core/utils/nested_projection_utils.h"
 #include "paimon/macros.h"
+#include "paimon/memory/memory_pool.h"
 
 namespace paimon {
 
-Status PrimaryKeyRealtimeStore::ValidateOptions(const CoreOptions& options,
-                                                const TableSchema& schema) {
-    if (options.GetBucket() <= 0) {
-        return Status::NotImplemented("PK realtime v1 requires fixed buckets");
-    }
-    if (options.GetMergeEngine() != MergeEngine::DEDUPLICATE) {
-        return Status::NotImplemented("PK realtime v1 supports only the DEDUPLICATE merge engine");
-    }
-    if (options.DataEvolutionEnabled()) {
-        return Status::NotImplemented("PK realtime v1 does not support data evolution");
-    }
-    if (!options.GetFieldsSequenceGroups().empty()) {
-        return Status::NotImplemented("PK realtime v1 does not support sequence groups");
-    }
-    if (options.IgnoreDelete() || options.PartialUpdateRemoveRecordOnDelete() ||
-        options.AggregationRemoveRecordOnDelete() ||
-        !options.GetPartialUpdateRemoveRecordOnSequenceGroup().empty()) {
-        return Status::NotImplemented("PK realtime v1 requires default delete behavior");
-    }
-    if (!options.GetSequenceField().empty()) {
-        return Status::NotImplemented("PK realtime v1 does not support sequence.field");
-    }
-    if (!options.SequenceFieldSortOrderIsAscending()) {
-        return Status::NotImplemented(
-            "PK realtime v1 supports only ascending sequence.field.sort-order");
-    }
-    if (options.NeedLookup() || options.DeletionVectorsEnabled() ||
-        options.GetChangelogProducer() != ChangelogProducer::NONE) {
-        return Status::NotImplemented("PK realtime v1 does not support lookup or early MOR");
-    }
-    PAIMON_ASSIGN_OR_RAISE(std::vector<DataField> primary_key_fields,
-                           schema.TrimmedPrimaryKeyFields());
-    for (const DataField& field : primary_key_fields) {
-        if (field.Type()->id() == arrow::Type::FLOAT || field.Type()->id() == arrow::Type::DOUBLE) {
-            return Status::NotImplemented(
-                "PK realtime v1 does not support FLOAT or DOUBLE primary keys");
-        }
-    }
-    if (options.GlobalIndexEnabled()) {
-        PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions definitions,
-                               PrimaryKeyIndexDefinitions::Create(schema));
-        if (!definitions.Definitions().empty()) {
-            return Status::NotImplemented("PK realtime v1 does not support global indexes");
-        }
-    }
-    return Status::OK();
-}
-
 namespace {
 
 Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
@@ -283,8 +233,11 @@ class StoredBatchReader final : public BatchReader {
 
 class PrimaryKeyRealtimeStore::Impl {
  public:
-    explicit Impl(std::shared_ptr<arrow::Schema> prepared_schema)
-        : prepared_schema_(std::move(prepared_schema)) {}
+    Impl(std::shared_ptr<arrow::Schema> prepared_schema, std::shared_ptr<MemoryPool> memory_pool,
+         std::shared_ptr<arrow::MemoryPool> arrow_pool)
+        : prepared_schema_(std::move(prepared_schema)),
+          memory_pool_(std::move(memory_pool)),
+          arrow_pool_(std::move(arrow_pool)) {}
 
     Status Write(RealtimeWriteBatch&& write_batch) {
         if (!write_batch.batch || !write_batch.batch->GetData()) {
@@ -369,7 +322,7 @@ class PrimaryKeyRealtimeStore::Impl {
                 PAIMON_ASSIGN_OR_RAISE(
                     std::shared_ptr<arrow::Array> projected,
                     AlignArrayByPaimonIds(batch.data, arrow::struct_(read_schema->fields()),
-                                          arrow::default_memory_pool()));
+                                          arrow_pool_.get()));
                 StoredBatch query_batch{checked_pointer_cast<arrow::StructArray>(projected),
                                         batch.offset_range, /*memory_usage=*/0};
                 readers.push_back(std::make_unique<StoredBatchReader>(query_batch));
@@ -399,6 +352,8 @@ class PrimaryKeyRealtimeStore::Impl {
 
  private:
     std::shared_ptr<arrow::Schema> prepared_schema_;
+    std::shared_ptr<MemoryPool> memory_pool_;
+    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
     mutable std::mutex mutex_;
     std::vector<StoredBatch> building_;
     std::vector<std::shared_ptr<Segment>> sealed_;
@@ -410,10 +365,15 @@ PrimaryKeyRealtimeStore::PrimaryKeyRealtimeStore(std::unique_ptr<Impl>&& impl)
 PrimaryKeyRealtimeStore::~PrimaryKeyRealtimeStore() = default;
 
 Result<std::shared_ptr<PrimaryKeyRealtimeStore>> PrimaryKeyRealtimeStore::Create(
-    const std::shared_ptr<arrow::Schema>& prepared_schema) {
+    const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::shared_ptr<MemoryPool>& memory_pool) {
     PAIMON_RETURN_NOT_OK(PreparedKeyValueReaderFactory::ValidateTransportSchema(prepared_schema));
-    return std::shared_ptr<PrimaryKeyRealtimeStore>(
-        new PrimaryKeyRealtimeStore(std::make_unique<Impl>(prepared_schema)));
+    if (!memory_pool) {
+        return Status::Invalid("PK real-time store memory pool is null");
+    }
+    std::shared_ptr<arrow::MemoryPool> arrow_pool = GetArrowPool(memory_pool);
+    return std::shared_ptr<PrimaryKeyRealtimeStore>(new PrimaryKeyRealtimeStore(
+        std::make_unique<Impl>(prepared_schema, memory_pool, std::move(arrow_pool))));
 }
 Status PrimaryKeyRealtimeStore::Write(RealtimeWriteBatch&& batch) {
     return impl_->Write(std::move(batch));
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.h b/src/paimon/core/realtime/primary_key_realtime_store.h
index df9a5f0c5..01e1926ab 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.h
+++ b/src/paimon/core/realtime/primary_key_realtime_store.h
@@ -29,15 +29,14 @@ class Schema;
 
 namespace paimon {
 
-class CoreOptions;
-class TableSchema;
+class MemoryPool;
 
 /// Internal in-memory implementation of the default primary-key `RealtimeStore`.
 class PrimaryKeyRealtimeStore final : public RealtimeStore {
  public:
     static Result<std::shared_ptr<PrimaryKeyRealtimeStore>> Create(
-        const std::shared_ptr<arrow::Schema>& prepared_schema);
-    static Status ValidateOptions(const CoreOptions& options, const TableSchema& schema);
+        const std::shared_ptr<arrow::Schema>& prepared_schema,
+        const std::shared_ptr<MemoryPool>& memory_pool);
 
     ~PrimaryKeyRealtimeStore() override;
 
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index 405c0d32a..7a1a3e27c 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -18,7 +18,10 @@
 
 #include "paimon/core/realtime/primary_key_realtime_store.h"
 
+#include <cstddef>
+#include <cstdint>
 #include <memory>
+#include <new>
 #include <optional>
 #include <string>
 #include <utility>
@@ -31,10 +34,9 @@
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
-#include "paimon/core/core_options.h"
-#include "paimon/core/schema/table_schema.h"
 #include "paimon/macros.h"
 #include "paimon/memory/memory_pool.h"
+#include "paimon/realtime/arrow_realtime_store_factory.h"
 #include "paimon/testing/utils/testharness.h"
 
 namespace paimon::test {
@@ -71,16 +73,6 @@ std::shared_ptr<arrow::Schema> NestedPreparedSchema() {
                                           arrow::field("items", arrow::list(arrow::int32()))}))))});
 }
 
-std::shared_ptr<TableSchema> PkSchema(
-    const std::shared_ptr<arrow::DataType>& key_type = arrow::int64(),
-    const std::map<std::string, std::string>& options = {}) {
-    return TableSchema::Create(
-               /*schema_id=*/0,
-               arrow::schema({arrow::field("id", key_type), arrow::field("value", arrow::utf8())}),
-               /*partition_keys=*/{}, /*primary_keys=*/{"id"}, options)
-        .value();
-}
-
 std::unique_ptr<RecordBatch> MakeBatch(const std::string& json) {
     std::shared_ptr<arrow::Array> array =
         arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(PreparedSchema()->fields()), json)
@@ -142,52 +134,50 @@ Result<std::string> ReadJson(const std::vector<std::unique_ptr<BatchReader>>& re
     return result->ToString();
 }
 
-TEST(PrimaryKeyRealtimeStoreOptionsTest, TestSupportedOptions) {
-    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
-    ASSERT_OK(PrimaryKeyRealtimeStore::ValidateOptions(options, *PkSchema()));
-}
+class TestingMemoryPool final : public MemoryPool {
+ public:
+    void* Malloc(uint64_t size, uint64_t alignment) override {
+        ++allocation_count;
+        if (reject_allocations) {
+            throw std::bad_alloc();
+        }
+        return delegate_->Malloc(size, alignment);
+    }
 
-TEST(PrimaryKeyRealtimeStoreOptionsTest, TestUnsupportedOptions) {
-    const std::string sequence_group =
-        std::string(Options::FIELDS_PREFIX) + ".value." + Options::SEQUENCE_GROUP;
-    const std::vector<std::map<std::string, std::string>> unsupported_options = {
-        {{Options::BUCKET, "0"}},
-        {{Options::BUCKET, "1"}, {Options::MERGE_ENGINE, "partial-update"}},
-        {{Options::BUCKET, "1"}, {Options::DATA_EVOLUTION_ENABLED, "true"}},
-        {{Options::BUCKET, "1"}, {sequence_group, "seq"}},
-        {{Options::BUCKET, "1"}, {Options::SEQUENCE_FIELD, "seq"}},
-        {{Options::BUCKET, "1"}, {Options::FORCE_LOOKUP, "true"}},
-        {{Options::BUCKET, "1"}, {Options::DELETION_VECTORS_ENABLED, "true"}},
-        {{Options::BUCKET, "1"}, {Options::CHANGELOG_PRODUCER, "input"}},
-    };
-    for (const std::map<std::string, std::string>& option_map : unsupported_options) {
-        ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
-        ASSERT_NOK(PrimaryKeyRealtimeStore::ValidateOptions(options, *PkSchema()));
+    void* Realloc(void* pointer, size_t old_size, size_t new_size, uint64_t alignment) override {
+        ++allocation_count;
+        if (reject_allocations) {
+            throw std::bad_alloc();
+        }
+        return delegate_->Realloc(pointer, old_size, new_size, alignment);
     }
-}
 
-TEST(PrimaryKeyRealtimeStoreOptionsTest, TestRejectsFloatingPrimaryKeys) {
-    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
-    ASSERT_NOK_WITH_MSG(
-        PrimaryKeyRealtimeStore::ValidateOptions(options, *PkSchema(arrow::float32())),
-        "FLOAT or DOUBLE primary keys");
-    ASSERT_NOK_WITH_MSG(
-        PrimaryKeyRealtimeStore::ValidateOptions(options, *PkSchema(arrow::float64())),
-        "FLOAT or DOUBLE primary keys");
-}
+    void Free(void* pointer, uint64_t size) override {
+        delegate_->Free(pointer, size);
+    }
 
-TEST(PrimaryKeyRealtimeStoreOptionsTest, TestRejectsEnabledGlobalIndex) {
-    const std::map<std::string, std::string> option_map = {{Options::BUCKET, "1"},
-                                                           {Options::PK_BTREE_INDEX_COLUMNS, "id"}};
-    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
-    ASSERT_NOK_WITH_MSG(
-        PrimaryKeyRealtimeStore::ValidateOptions(options, *PkSchema(arrow::int64(), option_map)),
-        "does not support global indexes");
-}
+    void Free(void* pointer, uint64_t size, uint64_t alignment) override {
+        delegate_->Free(pointer, size, alignment);
+    }
+
+    uint64_t CurrentUsage() const override {
+        return delegate_->CurrentUsage();
+    }
+
+    uint64_t MaxMemoryUsage() const override {
+        return delegate_->MaxMemoryUsage();
+    }
+
+    bool reject_allocations = false;
+    int64_t allocation_count = 0;
+
+ private:
+    std::unique_ptr<MemoryPool> delegate_ = GetMemoryPool();
+};
 
 TEST(PrimaryKeyRealtimeStoreTest, TestWriteAndSealValidation) {
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(PreparedSchema()));
+                         PrimaryKeyRealtimeStore::Create(PreparedSchema(), GetDefaultPool()));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
                          store->SealForCommit());
     ASSERT_FALSE(segment.has_value());
@@ -232,14 +222,15 @@ TEST(PrimaryKeyRealtimeStoreTest, TestBadTransportPrefix) {
     invalid_fields.push_back(std::move(wrong_offset_id));
 
     for (const arrow::FieldVector& fields : invalid_fields) {
-        ASSERT_NOK_WITH_MSG(PrimaryKeyRealtimeStore::Create(arrow::schema(fields)),
-                            "prepared schema field");
+        ASSERT_NOK_WITH_MSG(
+            PrimaryKeyRealtimeStore::Create(arrow::schema(fields), GetDefaultPool()),
+            "prepared schema field");
     }
 }
 
 TEST(PrimaryKeyRealtimeStoreTest, TestCommitReaderPerStoredBatch) {
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(PreparedSchema()));
+                         PrimaryKeyRealtimeStore::Create(PreparedSchema(), GetDefaultPool()));
     ASSERT_OK(store->Write(RealtimeWriteBatch{
         MakeBatch(R"([[1, 6, 1, 1, "before"], [0, 5, 0, 3, "three"]])"), OffsetRange(0, 2)}));
     ASSERT_OK(store->Write(
@@ -265,7 +256,7 @@ TEST(PrimaryKeyRealtimeStoreTest, TestCommitReaderPerStoredBatch) {
 TEST(PrimaryKeyRealtimeStoreTest, TestCommitReaderExportsZeroOffsets) {
     std::shared_ptr<arrow::Schema> schema = NestedPreparedSchema();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(schema));
+                         PrimaryKeyRealtimeStore::Create(schema, GetDefaultPool()));
     ASSERT_OK(store->Write(RealtimeWriteBatch{
         MakeBatch(schema, R"([[0, 1, 0, 1, ["one", [1, 2]]], [0, 2, 1, 2, ["two", [3, 4]]]])"),
         OffsetRange(0, 2)}));
@@ -286,7 +277,7 @@ TEST(PrimaryKeyRealtimeStoreTest, TestCommitReaderExportsZeroOffsets) {
 
 TEST(PrimaryKeyRealtimeStoreTest, TestCloseUnreadBatchReaders) {
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(PreparedSchema()));
+                         PrimaryKeyRealtimeStore::Create(PreparedSchema(), GetDefaultPool()));
     ASSERT_OK(
         store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 10, 0, 1, "a"]])"), OffsetRange(0, 1)}));
     ASSERT_OK(
@@ -306,7 +297,7 @@ TEST(PrimaryKeyRealtimeStoreTest, TestCloseUnreadBatchReaders) {
 
 TEST(PrimaryKeyRealtimeStoreTest, TestReclaimKeepsReadView) {
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(PreparedSchema()));
+                         PrimaryKeyRealtimeStore::Create(PreparedSchema(), GetDefaultPool()));
     ASSERT_OK(
         store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 0, 4, 1, "one"]])"), OffsetRange(4, 5)}));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
@@ -318,7 +309,7 @@ TEST(PrimaryKeyRealtimeStoreTest, TestReclaimKeepsReadView) {
 
 TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderPerStoredBatch) {
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(PreparedSchema()));
+                         PrimaryKeyRealtimeStore::Create(PreparedSchema(), GetDefaultPool()));
     ASSERT_OK(
         store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 1, 0, 2, "two"]])"), OffsetRange(0, 1)}));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
@@ -342,7 +333,7 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderPerStoredBatch) {
 TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderProjectsTopLevelFieldsById) {
     const std::shared_ptr<arrow::Schema> stored_schema = PreparedSchema();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(stored_schema));
+                         PrimaryKeyRealtimeStore::Create(stored_schema, GetDefaultPool()));
     ASSERT_OK(store->Write(RealtimeWriteBatch{
         MakeSlicedBatch(stored_schema,
                         R"([[0, 1, 0, 6, "six"], [0, 2, 1, 7, "seven"], [0, 3, 2, 8, "eight"]])", 1,
@@ -374,6 +365,34 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderProjectsTopLevelFieldsById) {
     ASSERT_TRUE(projected->field(4)->IsNull(0));
 }
 
+TEST(PrimaryKeyRealtimeStoreTest, TestQuerySchemaAlignmentUsesCallerPool) {
+    const std::shared_ptr<arrow::Schema> stored_schema = PreparedSchema();
+    std::shared_ptr<TestingMemoryPool> pool = std::make_shared<TestingMemoryPool>();
+    auto write_schema = std::make_unique<ArrowSchema>();
+    ASSERT_TRUE(arrow::ExportSchema(*stored_schema, write_schema.get()).ok());
+    RealtimeStoreCreateRequest request{std::move(write_schema),
+                                       /*options=*/{}, pool, RealtimeStoreMode::PRIMARY_KEY};
+    ArrowRealtimeStoreFactory factory;
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeStore> store, factory.Create(std::move(request)));
+    ASSERT_OK(store->Write(
+        RealtimeWriteBatch{MakeBatch(R"([[0, 1, 0, 7, "seven"]])"), OffsetRange(0, 1)}));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
+
+    arrow::FieldVector requested_fields = stored_schema->fields();
+    requested_fields.push_back(FieldWithId("added", arrow::int32(), 2));
+    std::shared_ptr<arrow::Schema> requested_schema = arrow::schema(std::move(requested_fields));
+    auto c_schema = std::make_unique<ArrowSchema>();
+    ASSERT_TRUE(arrow::ExportSchema(*requested_schema, c_schema.get()).ok());
+    RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+
+    const int64_t allocations_before_query = pool->allocation_count;
+    pool->reject_allocations = true;
+    ASSERT_NOK_WITH_MSG(store->CreateQueryReaders(view, /*offset_begin=*/0, context),
+                        "Out of memory");
+    ASSERT_GT(pool->allocation_count, allocations_before_query);
+}
+
 TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderAlignsNestedFieldsById) {
     const std::shared_ptr<arrow::Field> stored_a = FieldWithId("a", arrow::int32(), 10);
     const std::shared_ptr<arrow::Field> stored_b = FieldWithId("b", arrow::int32(), 11);
@@ -389,7 +408,7 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderAlignsNestedFieldsById) {
         FieldWithId("attrs", arrow::map(arrow::utf8(), arrow::struct_({stored_x, stored_y})), 2)};
     std::shared_ptr<arrow::Schema> stored_schema = arrow::schema(std::move(stored_fields));
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(stored_schema));
+                         PrimaryKeyRealtimeStore::Create(stored_schema, GetDefaultPool()));
     ASSERT_OK(store->Write(RealtimeWriteBatch{
         MakeSlicedBatch(
             stored_schema,
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index 692eff1bd..bacbfa5e2 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -19,12 +19,14 @@
 #include "paimon/core/realtime/realtime_primary_key_writer.h"
 
 #include <limits>
+#include <new>
 #include <optional>
 #include <utility>
 #include <vector>
 
 #include "arrow/api.h"
 #include "arrow/c/bridge.h"
+#include "arrow/c/helpers.h"
 #include "arrow/compute/api.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
@@ -52,20 +54,31 @@ struct PreparedArrayPrivateData {
 };
 
 void ReleasePreparedArray(ArrowArray* array) {
-    auto* data = static_cast<PreparedArrayPrivateData*>(array->private_data);
+    std::unique_ptr<PreparedArrayPrivateData> data(
+        static_cast<PreparedArrayPrivateData*>(array->private_data));
     array->release = data->release;
     array->private_data = data->private_data;
     array->release(array);
-    delete data;
 }
 
 Status RetainPreparedArrayPool(ArrowArray* array,
                                const std::shared_ptr<arrow::MemoryPool>& arrow_pool) {
-    if (!array || !array->release || !arrow_pool) {
+    if (!array || !array->release) {
         return Status::Invalid("cannot retain prepared batch memory pool");
     }
-    array->private_data =
-        new PreparedArrayPrivateData{array->release, array->private_data, arrow_pool};
+    if (!arrow_pool) {
+        ArrowArrayRelease(array);
+        return Status::Invalid("cannot retain prepared batch memory pool");
+    }
+    std::unique_ptr<PreparedArrayPrivateData> data;
+    try {
+        data = std::make_unique<PreparedArrayPrivateData>(
+            PreparedArrayPrivateData{array->release, array->private_data, arrow_pool});
+    } catch (const std::bad_alloc&) {
+        ArrowArrayRelease(array);
+        return Status::OutOfMemory("failed to retain prepared batch memory pool");
+    }
+    array->private_data = data.release();
     array->release = ReleasePreparedArray;
     return Status::OK();
 }
diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp
index 70ab0cb8e..85a9e83b3 100644
--- a/src/paimon/core/table/source/table_scan.cpp
+++ b/src/paimon/core/table/source/table_scan.cpp
@@ -41,7 +41,7 @@
 #include "paimon/core/operation/data_evolution_file_store_scan.h"
 #include "paimon/core/operation/file_store_scan.h"
 #include "paimon/core/operation/key_value_file_store_scan.h"
-#include "paimon/core/realtime/primary_key_realtime_store.h"
+#include "paimon/core/realtime/framework/primary_key_realtime_validator.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
 #include "paimon/core/schema/schema_manager.h"
 #include "paimon/core/schema/schema_validation.h"
@@ -237,7 +237,8 @@ Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions&
             return Status::NotImplemented(
                 "PK real-time union read does not support read-optimized scans");
         }
-        PAIMON_RETURN_NOT_OK(PrimaryKeyRealtimeStore::ValidateOptions(core_options, table_schema));
+        PAIMON_RETURN_NOT_OK(
+            PrimaryKeyRealtimeValidator::ValidateOptions(core_options, table_schema));
     }
     if (context.IsStreamingMode()) {
         return Status::Invalid("real-time union read currently supports batch scans only");

From 52ee6bd5f95987c1b1ebad8ccf2bc8c0ddee28d0 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Tue, 25 Aug 2026 23:40:14 +0800
Subject: [PATCH 36/62] fix(realtime): harden query schema alignment

---
 src/paimon/CMakeLists.txt                     |   1 +
 .../core/realtime/arrow_array_pool_holder.cpp |  69 ++++++++
 .../core/realtime/arrow_array_pool_holder.h   |  36 ++++
 .../realtime/primary_key_realtime_store.cpp   |  75 +++++---
 .../primary_key_realtime_store_test.cpp       | 166 ++++++++++++++++--
 .../realtime/realtime_primary_key_writer.cpp  |  41 +----
 6 files changed, 316 insertions(+), 72 deletions(-)
 create mode 100644 src/paimon/core/realtime/arrow_array_pool_holder.cpp
 create mode 100644 src/paimon/core/realtime/arrow_array_pool_holder.h

diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index 7ff15100b..fb702bfea 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -380,6 +380,7 @@ set(PAIMON_CORE_SRCS
     core/manifest/snapshot_live_manifest_entries.cpp
     core/operation/write_context.cpp
     core/operation/write_restore.cpp
+    core/realtime/arrow_array_pool_holder.cpp
     core/realtime/arrow_realtime_store.cpp
     core/realtime/arrow_realtime_store_factory.cpp
     core/realtime/framework/primary_key_realtime_validator.cpp
diff --git a/src/paimon/core/realtime/arrow_array_pool_holder.cpp b/src/paimon/core/realtime/arrow_array_pool_holder.cpp
new file mode 100644
index 000000000..97a01dd19
--- /dev/null
+++ b/src/paimon/core/realtime/arrow_array_pool_holder.cpp
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/realtime/arrow_array_pool_holder.h"
+
+#include <memory>
+#include <new>
+
+#include "arrow/c/abi.h"
+#include "arrow/c/helpers.h"
+#include "arrow/memory_pool.h"
+
+namespace paimon {
+namespace {
+
+struct ArrowArrayPrivateData {
+    void (*release)(ArrowArray*);
+    void* private_data;
+    std::shared_ptr<arrow::MemoryPool> arrow_pool;
+};
+
+void ReleaseArrowArray(ArrowArray* array) {
+    std::unique_ptr<ArrowArrayPrivateData> data(
+        static_cast<ArrowArrayPrivateData*>(array->private_data));
+    array->release = data->release;
+    array->private_data = data->private_data;
+    array->release(array);
+}
+
+}  // namespace
+
+Status RetainArrowArrayMemoryPool(ArrowArray* array,
+                                  const std::shared_ptr<arrow::MemoryPool>& arrow_pool) {
+    if (!array || !array->release) {
+        return Status::Invalid("cannot retain Arrow array memory pool");
+    }
+    if (!arrow_pool) {
+        ArrowArrayRelease(array);
+        return Status::Invalid("cannot retain Arrow array memory pool");
+    }
+    std::unique_ptr<ArrowArrayPrivateData> data;
+    try {
+        data = std::make_unique<ArrowArrayPrivateData>(
+            ArrowArrayPrivateData{array->release, array->private_data, arrow_pool});
+    } catch (const std::bad_alloc&) {
+        ArrowArrayRelease(array);
+        return Status::OutOfMemory("failed to retain Arrow array memory pool");
+    }
+    array->private_data = data.release();
+    array->release = ReleaseArrowArray;
+    return Status::OK();
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/realtime/arrow_array_pool_holder.h b/src/paimon/core/realtime/arrow_array_pool_holder.h
new file mode 100644
index 000000000..05f9ed9dc
--- /dev/null
+++ b/src/paimon/core/realtime/arrow_array_pool_holder.h
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <memory>
+
+#include "paimon/status.h"
+
+struct ArrowArray;
+
+namespace arrow {
+class MemoryPool;
+}  // namespace arrow
+
+namespace paimon {
+
+Status RetainArrowArrayMemoryPool(ArrowArray* array,
+                                  const std::shared_ptr<arrow::MemoryPool>& arrow_pool);
+
+}  // namespace paimon
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index 88e78680a..cbaef65e2 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -32,6 +32,7 @@
 #include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
+#include "paimon/core/realtime/arrow_array_pool_holder.h"
 #include "paimon/core/realtime/prepared_key_value_reader.h"
 #include "paimon/core/utils/nested_projection_utils.h"
 #include "paimon/macros.h"
@@ -41,9 +42,14 @@ namespace paimon {
 
 namespace {
 
-Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
-    const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& read_type,
-    arrow::MemoryPool* pool);
+struct AlignedArray {
+    std::shared_ptr<arrow::Array> array;
+    bool uses_arrow_pool;
+};
+
+Result<AlignedArray> AlignArrayByPaimonIds(const std::shared_ptr<arrow::Array>& array,
+                                           const std::shared_ptr<arrow::DataType>& read_type,
+                                           arrow::MemoryPool* pool);
 
 bool TypesExactlyEqual(const std::shared_ptr<arrow::DataType>& data_type,
                        const std::shared_ptr<arrow::DataType>& read_type) {
@@ -59,7 +65,7 @@ bool TypesExactlyEqual(const std::shared_ptr<arrow::DataType>& data_type,
     return true;
 }
 
-Result<std::shared_ptr<arrow::Array>> AlignStructArrayByPaimonIds(
+Result<AlignedArray> AlignStructArrayByPaimonIds(
     const std::shared_ptr<arrow::StructArray>& array,
     const std::shared_ptr<arrow::StructType>& read_type, arrow::MemoryPool* pool) {
     const std::shared_ptr<arrow::StructType> data_type =
@@ -78,6 +84,7 @@ Result<std::shared_ptr<arrow::Array>> AlignStructArrayByPaimonIds(
     requested_field_ids.reserve(read_type->num_fields());
     std::vector<std::shared_ptr<arrow::ArrayData>> children;
     children.reserve(read_type->num_fields());
+    bool uses_arrow_pool = false;
     for (const std::shared_ptr<arrow::Field>& read_field : read_type->fields()) {
         PAIMON_ASSIGN_OR_RAISE(int32_t field_id,
                                NestedProjectionUtils::GetPaimonFieldId(read_field));
@@ -87,30 +94,38 @@ Result<std::shared_ptr<arrow::Array>> AlignStructArrayByPaimonIds(
         }
         const auto data_iter = data_field_indexes.find(field_id);
         if (data_iter == data_field_indexes.end()) {
+            if (!read_field->nullable()) {
+                return Status::Invalid(fmt::format(
+                    "requested non-nullable field '{}' with id {} is absent from stored schema",
+                    read_field->name(), field_id));
+            }
             PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
                 std::shared_ptr<arrow::Array> null_child,
                 arrow::MakeArrayOfNull(read_field->type(), array->offset() + array->length(),
                                        pool));
             children.push_back(null_child->data());
+            uses_arrow_pool = true;
             continue;
         }
         std::shared_ptr<arrow::Array> child =
             arrow::MakeArray(array->data()->child_data[data_iter->second]);
-        PAIMON_ASSIGN_OR_RAISE(child, AlignArrayByPaimonIds(child, read_field->type(), pool));
-        children.push_back(child->data());
+        PAIMON_ASSIGN_OR_RAISE(AlignedArray aligned_child,
+                               AlignArrayByPaimonIds(child, read_field->type(), pool));
+        children.push_back(aligned_child.array->data());
+        uses_arrow_pool = uses_arrow_pool || aligned_child.uses_arrow_pool;
     }
 
     std::shared_ptr<arrow::ArrayData> aligned = array->data()->Copy();
     aligned->type = read_type;
     aligned->child_data = std::move(children);
-    return arrow::MakeArray(std::move(aligned));
+    return AlignedArray{arrow::MakeArray(std::move(aligned)), uses_arrow_pool};
 }
 
-Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
-    const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& read_type,
-    arrow::MemoryPool* pool) {
+Result<AlignedArray> AlignArrayByPaimonIds(const std::shared_ptr<arrow::Array>& array,
+                                           const std::shared_ptr<arrow::DataType>& read_type,
+                                           arrow::MemoryPool* pool) {
     if (TypesExactlyEqual(array->type(), read_type)) {
-        return array;
+        return AlignedArray{array, false};
     }
     if (array->type_id() != read_type->id()) {
         return Status::Invalid(fmt::format("stored value type {} does not match requested type {}",
@@ -125,11 +140,13 @@ Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
             std::shared_ptr<arrow::Array> values =
                 checked_pointer_cast<arrow::ListArray>(array)->values();
             PAIMON_ASSIGN_OR_RAISE(
-                values, AlignArrayByPaimonIds(values, read_type->field(0)->type(), pool));
+                AlignedArray aligned_values,
+                AlignArrayByPaimonIds(values, read_type->field(0)->type(), pool));
             std::shared_ptr<arrow::ArrayData> aligned = array->data()->Copy();
             aligned->type = read_type;
-            aligned->child_data = {values->data()};
-            return arrow::MakeArray(std::move(aligned));
+            aligned->child_data = {aligned_values.array->data()};
+            return AlignedArray{arrow::MakeArray(std::move(aligned)),
+                                aligned_values.uses_arrow_pool};
         }
         case arrow::Type::MAP: {
             const std::shared_ptr<arrow::MapArray> map =
@@ -137,17 +154,19 @@ Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
             const std::shared_ptr<arrow::MapType> map_type =
                 checked_pointer_cast<arrow::MapType>(read_type);
             std::shared_ptr<arrow::Array> keys = map->keys();
-            PAIMON_ASSIGN_OR_RAISE(keys, AlignArrayByPaimonIds(keys, map_type->key_type(), pool));
+            PAIMON_ASSIGN_OR_RAISE(AlignedArray aligned_keys,
+                                   AlignArrayByPaimonIds(keys, map_type->key_type(), pool));
             std::shared_ptr<arrow::Array> items = map->items();
-            PAIMON_ASSIGN_OR_RAISE(items,
+            PAIMON_ASSIGN_OR_RAISE(AlignedArray aligned_items,
                                    AlignArrayByPaimonIds(items, map_type->item_type(), pool));
             std::shared_ptr<arrow::ArrayData> entries = array->data()->child_data[0]->Copy();
             entries->type = arrow::struct_({map_type->key_field(), map_type->item_field()});
-            entries->child_data = {keys->data(), items->data()};
+            entries->child_data = {aligned_keys.array->data(), aligned_items.array->data()};
             std::shared_ptr<arrow::ArrayData> aligned = array->data()->Copy();
             aligned->type = read_type;
             aligned->child_data = {std::move(entries)};
-            return arrow::MakeArray(std::move(aligned));
+            return AlignedArray{arrow::MakeArray(std::move(aligned)),
+                                aligned_keys.uses_arrow_pool || aligned_items.uses_arrow_pool};
         }
         default:
             return Status::Invalid(
@@ -203,8 +222,11 @@ class ReadView final : public RealtimeReadView {
 
 class StoredBatchReader final : public BatchReader {
  public:
-    explicit StoredBatchReader(const StoredBatch& batch)
-        : data_(batch.data), metrics_(std::make_shared<MetricsImpl>()) {}
+    explicit StoredBatchReader(const StoredBatch& batch,
+                               std::shared_ptr<arrow::MemoryPool> arrow_pool = nullptr)
+        : arrow_pool_(std::move(arrow_pool)),
+          data_(batch.data),
+          metrics_(std::make_shared<MetricsImpl>()) {}
 
     Result<ReadBatch> NextBatch() override {
         if (!data_) {
@@ -213,7 +235,11 @@ class StoredBatchReader final : public BatchReader {
         auto array = std::make_unique<ArrowArray>();
         auto schema = std::make_unique<ArrowSchema>();
         PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*data_, array.get(), schema.get()));
+        if (arrow_pool_) {
+            PAIMON_RETURN_NOT_OK(RetainArrowArrayMemoryPool(array.get(), arrow_pool_));
+        }
         data_.reset();
+        arrow_pool_.reset();
         return ReadBatch(std::move(array), std::move(schema));
     }
 
@@ -222,9 +248,11 @@ class StoredBatchReader final : public BatchReader {
     }
     void Close() override {
         data_.reset();
+        arrow_pool_.reset();
     }
 
  private:
+    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
     std::shared_ptr<arrow::StructArray> data_;
     std::shared_ptr<Metrics> metrics_;
 };
@@ -320,12 +348,13 @@ class PrimaryKeyRealtimeStore::Impl {
         for (const std::shared_ptr<Segment>& segment : typed->Segments()) {
             for (const StoredBatch& batch : segment->Batches()) {
                 PAIMON_ASSIGN_OR_RAISE(
-                    std::shared_ptr<arrow::Array> projected,
+                    AlignedArray projected,
                     AlignArrayByPaimonIds(batch.data, arrow::struct_(read_schema->fields()),
                                           arrow_pool_.get()));
-                StoredBatch query_batch{checked_pointer_cast<arrow::StructArray>(projected),
+                StoredBatch query_batch{checked_pointer_cast<arrow::StructArray>(projected.array),
                                         batch.offset_range, /*memory_usage=*/0};
-                readers.push_back(std::make_unique<StoredBatchReader>(query_batch));
+                readers.push_back(std::make_unique<StoredBatchReader>(
+                    query_batch, projected.uses_arrow_pool ? arrow_pool_ : nullptr));
             }
         }
         return readers;
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index 7a1a3e27c..a63d79eec 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -44,8 +44,10 @@ namespace {
 
 std::shared_ptr<arrow::Field> FieldWithId(const std::string& name,
                                           const std::shared_ptr<arrow::DataType>& type,
-                                          int32_t field_id) {
-    return DataField::ConvertDataFieldToArrowField(DataField(field_id, arrow::field(name, type)));
+                                          int32_t field_id, bool nullable = true) {
+    return DataField::ConvertDataFieldToArrowField(
+               DataField(field_id, arrow::field(name, type, nullable)))
+        ->WithNullable(nullable);
 }
 
 std::shared_ptr<arrow::Schema> PreparedSchema() {
@@ -365,6 +367,26 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderProjectsTopLevelFieldsById) {
     ASSERT_TRUE(projected->field(4)->IsNull(0));
 }
 
+TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderRejectsMissingNonNullableTopLevelField) {
+    const std::shared_ptr<arrow::Schema> stored_schema = PreparedSchema();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                         PrimaryKeyRealtimeStore::Create(stored_schema, GetDefaultPool()));
+    ASSERT_OK(store->Write(
+        RealtimeWriteBatch{MakeBatch(R"([[0, 1, 0, 7, "seven"]])"), OffsetRange(0, 1)}));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
+
+    arrow::FieldVector requested_fields = stored_schema->fields();
+    requested_fields.push_back(
+        FieldWithId("required_added", arrow::int32(), 2, /*nullable=*/false));
+    auto c_schema = std::make_unique<ArrowSchema>();
+    ASSERT_TRUE(
+        arrow::ExportSchema(*arrow::schema(std::move(requested_fields)), c_schema.get()).ok());
+    RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+    ASSERT_NOK_WITH_MSG(store->CreateQueryReaders(view, /*offset_begin=*/0, context),
+                        "requested non-nullable field 'required_added' with id 2 is absent");
+}
+
 TEST(PrimaryKeyRealtimeStoreTest, TestQuerySchemaAlignmentUsesCallerPool) {
     const std::shared_ptr<arrow::Schema> stored_schema = PreparedSchema();
     std::shared_ptr<TestingMemoryPool> pool = std::make_shared<TestingMemoryPool>();
@@ -386,14 +408,81 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQuerySchemaAlignmentUsesCallerPool) {
     RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
                                  /*enable_predicate_pushdown=*/false};
 
-    const int64_t allocations_before_query = pool->allocation_count;
+    std::vector<arrow::FieldVector> zero_copy_schemas;
+    zero_copy_schemas.push_back(stored_schema->fields());
+    arrow::FieldVector reordered_fields(stored_schema->fields().begin(),
+                                        stored_schema->fields().begin() + 3);
+    reordered_fields.push_back(FieldWithId("renamed_value", arrow::utf8(), 1));
+    reordered_fields.push_back(FieldWithId("renamed_id", arrow::int64(), 0));
+    zero_copy_schemas.push_back(std::move(reordered_fields));
     pool->reject_allocations = true;
+    for (const arrow::FieldVector& fields : zero_copy_schemas) {
+        auto zero_copy_schema = std::make_unique<ArrowSchema>();
+        ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(fields), zero_copy_schema.get()).ok());
+        RealtimeQueryContext zero_copy_context{zero_copy_schema.get(), /*predicate=*/nullptr,
+                                               /*enable_predicate_pushdown=*/false};
+        const int64_t allocations_before_query = pool->allocation_count;
+        ASSERT_OK_AND_ASSIGN(
+            std::vector<std::unique_ptr<BatchReader>> readers,
+            store->CreateQueryReaders(view, /*offset_begin=*/0, zero_copy_context));
+        ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, readers[0]->NextBatch());
+        ASSERT_TRUE(arrow::ImportArray(batch.first.get(), batch.second.get()).ok());
+        ASSERT_EQ(allocations_before_query, pool->allocation_count);
+    }
+
+    const int64_t allocations_before_query = pool->allocation_count;
     ASSERT_NOK_WITH_MSG(store->CreateQueryReaders(view, /*offset_begin=*/0, context),
                         "Out of memory");
     ASSERT_GT(pool->allocation_count, allocations_before_query);
 }
 
+TEST(PrimaryKeyRealtimeStoreTest, TestQueryAlignmentPoolOutlivesStoreReaderAndExport) {
+    const std::shared_ptr<arrow::Schema> stored_schema = PreparedSchema();
+    std::shared_ptr<TestingMemoryPool> pool = std::make_shared<TestingMemoryPool>();
+    std::weak_ptr<TestingMemoryPool> pool_lifetime = pool;
+    auto write_schema = std::make_unique<ArrowSchema>();
+    ASSERT_TRUE(arrow::ExportSchema(*stored_schema, write_schema.get()).ok());
+    RealtimeStoreCreateRequest request{std::move(write_schema),
+                                       /*options=*/{}, pool, RealtimeStoreMode::PRIMARY_KEY};
+    ArrowRealtimeStoreFactory factory;
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeStore> store, factory.Create(std::move(request)));
+    request.memory_pool.reset();
+    ASSERT_OK(store->Write(
+        RealtimeWriteBatch{MakeBatch(R"([[0, 1, 0, 7, "seven"]])"), OffsetRange(0, 1)}));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
+
+    arrow::FieldVector requested_fields = stored_schema->fields();
+    requested_fields.push_back(FieldWithId("added", arrow::int32(), 2));
+    auto c_schema = std::make_unique<ArrowSchema>();
+    ASSERT_TRUE(
+        arrow::ExportSchema(*arrow::schema(std::move(requested_fields)), c_schema.get()).ok());
+    RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store->CreateQueryReaders(view, /*offset_begin=*/0, context));
+    ASSERT_EQ(1, readers.size());
+    ASSERT_GT(pool->allocation_count, 0);
+
+    view.reset();
+    store.reset();
+    pool.reset();
+    ASSERT_FALSE(pool_lifetime.expired());
+
+    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, readers[0]->NextBatch());
+    ASSERT_FALSE(BatchReader::IsEofBatch(batch));
+    arrow::Result<std::shared_ptr<arrow::Array>> import_result =
+        arrow::ImportArray(batch.first.get(), batch.second.get());
+    ASSERT_TRUE(import_result.ok()) << import_result.status().ToString();
+    std::shared_ptr<arrow::Array> imported = std::move(import_result).ValueOrDie();
+    readers.clear();
+    ASSERT_FALSE(pool_lifetime.expired());
+    imported.reset();
+    ASSERT_TRUE(pool_lifetime.expired());
+}
+
 TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderAlignsNestedFieldsById) {
+    const std::shared_ptr<arrow::Field> stored_profile_a =
+        FieldWithId("profile_a", arrow::int32(), 30);
     const std::shared_ptr<arrow::Field> stored_a = FieldWithId("a", arrow::int32(), 10);
     const std::shared_ptr<arrow::Field> stored_b = FieldWithId("b", arrow::int32(), 11);
     const std::shared_ptr<arrow::Field> stored_x = FieldWithId("x", arrow::int32(), 20);
@@ -404,19 +493,24 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderAlignsNestedFieldsById) {
             ->WithNullable(false),
         DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()),
         FieldWithId("id", arrow::int64(), 0),
-        FieldWithId("items", arrow::list(arrow::struct_({stored_a, stored_b})), 1),
-        FieldWithId("attrs", arrow::map(arrow::utf8(), arrow::struct_({stored_x, stored_y})), 2)};
+        FieldWithId("profile", arrow::struct_({stored_profile_a}), 1),
+        FieldWithId("items", arrow::list(arrow::struct_({stored_a, stored_b})), 2),
+        FieldWithId("attrs", arrow::map(arrow::utf8(), arrow::struct_({stored_x, stored_y})), 3)};
     std::shared_ptr<arrow::Schema> stored_schema = arrow::schema(std::move(stored_fields));
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
                          PrimaryKeyRealtimeStore::Create(stored_schema, GetDefaultPool()));
     ASSERT_OK(store->Write(RealtimeWriteBatch{
         MakeSlicedBatch(
             stored_schema,
-            R"([[0, 1, 0, 6, [[1, 2]], [["before", [3, 4]]]], [0, 2, 1, 7, [[100, 200], null], [["k1", [7, 8]], ["k2", null]]], [0, 3, 2, 8, [[9, 10]], [["after", [11, 12]]]]])",
+            R"([[0, 1, 0, 6, [5], [[1, 2]], [["before", [3, 4]]]], [0, 2, 1, 7, [50], [[100, 200], null], [["k1", [7, 8]], ["k2", null]]], [0, 3, 2, 8, [500], [[9, 10]], [["after", [11, 12]]]]])",
             1, 1),
         OffsetRange(0, 1)}));
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
 
+    const std::shared_ptr<arrow::Field> requested_profile_missing =
+        FieldWithId("added_profile", arrow::int32(), 31);
+    const std::shared_ptr<arrow::Field> requested_profile_a =
+        FieldWithId("renamed_profile_a", arrow::int32(), 30);
     const std::shared_ptr<arrow::Field> requested_b = FieldWithId("renamed_b", arrow::int32(), 11);
     const std::shared_ptr<arrow::Field> requested_a = FieldWithId("renamed_a", arrow::int32(), 10);
     const std::shared_ptr<arrow::Field> requested_item_missing =
@@ -427,14 +521,16 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderAlignsNestedFieldsById) {
         FieldWithId("added_attr", arrow::int32(), 22);
     arrow::FieldVector requested_fields(stored_schema->fields().begin(),
                                         stored_schema->fields().begin() + 3);
+    requested_fields.push_back(FieldWithId(
+        "renamed_profile", arrow::struct_({requested_profile_missing, requested_profile_a}), 1));
     requested_fields.push_back(FieldWithId(
         "renamed_items",
-        arrow::list(arrow::struct_({requested_b, requested_item_missing, requested_a})), 1));
+        arrow::list(arrow::struct_({requested_b, requested_item_missing, requested_a})), 2));
     requested_fields.push_back(
         FieldWithId("renamed_attrs",
                     arrow::map(arrow::utf8(),
                                arrow::struct_({requested_y, requested_attr_missing, requested_x})),
-                    2));
+                    3));
     std::shared_ptr<arrow::Schema> requested_schema = arrow::schema(std::move(requested_fields));
     auto c_schema = std::make_unique<ArrowSchema>();
     ASSERT_TRUE(arrow::ExportSchema(*requested_schema, c_schema.get()).ok());
@@ -449,8 +545,12 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderAlignsNestedFieldsById) {
     std::shared_ptr<arrow::Array> array = std::move(import_result).ValueOrDie();
     ASSERT_TRUE(array->type()->Equals(arrow::struct_(requested_schema->fields())));
     std::shared_ptr<arrow::StructArray> projected = checked_pointer_cast<arrow::StructArray>(array);
+    const std::shared_ptr<arrow::StructArray> profile =
+        checked_pointer_cast<arrow::StructArray>(projected->field(3));
+    ASSERT_TRUE(profile->field(0)->IsNull(0));
+    ASSERT_EQ(50, checked_pointer_cast<arrow::Int32Array>(profile->field(1))->Value(0));
     const std::shared_ptr<arrow::ListArray> items =
-        checked_pointer_cast<arrow::ListArray>(projected->field(3));
+        checked_pointer_cast<arrow::ListArray>(projected->field(4));
     const std::shared_ptr<arrow::StructArray> item_values =
         checked_pointer_cast<arrow::StructArray>(items->value_slice(0));
     ASSERT_EQ(200, checked_pointer_cast<arrow::Int32Array>(item_values->field(0))->Value(0));
@@ -459,7 +559,7 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderAlignsNestedFieldsById) {
     ASSERT_TRUE(item_values->IsNull(1));
 
     const std::shared_ptr<arrow::MapArray> attrs =
-        checked_pointer_cast<arrow::MapArray>(projected->field(4));
+        checked_pointer_cast<arrow::MapArray>(projected->field(5));
     const int64_t attr_offset = attrs->value_offset(0);
     const int64_t attr_length = attrs->value_length(0);
     const std::shared_ptr<arrow::StringArray> attr_keys =
@@ -473,5 +573,51 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderAlignsNestedFieldsById) {
     ASSERT_TRUE(attr_values->IsNull(1));
 }
 
+TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderRejectsMissingNonNullableNestedFields) {
+    const std::shared_ptr<arrow::Field> stored_profile_a =
+        FieldWithId("profile_a", arrow::int32(), 30);
+    const std::shared_ptr<arrow::Field> stored_item_a = FieldWithId("item_a", arrow::int32(), 10);
+    const std::shared_ptr<arrow::Field> stored_attr_a = FieldWithId("attr_a", arrow::int32(), 20);
+    arrow::FieldVector stored_fields = {
+        DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
+        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
+            ->WithNullable(false),
+        DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()),
+        FieldWithId("profile", arrow::struct_({stored_profile_a}), 1),
+        FieldWithId("items", arrow::list(arrow::struct_({stored_item_a})), 2),
+        FieldWithId("attrs", arrow::map(arrow::utf8(), arrow::struct_({stored_attr_a})), 3)};
+    const std::shared_ptr<arrow::Schema> stored_schema = arrow::schema(std::move(stored_fields));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                         PrimaryKeyRealtimeStore::Create(stored_schema, GetDefaultPool()));
+    ASSERT_OK(store->Write(
+        RealtimeWriteBatch{MakeBatch(stored_schema, R"([[0, 1, 0, [5], [[10]], [["key", [20]]]]])"),
+                           OffsetRange(0, 1)}));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
+
+    const std::shared_ptr<arrow::Field> required =
+        FieldWithId("required_nested", arrow::int32(), 99, /*nullable=*/false);
+    std::vector<arrow::FieldVector> requested_schemas;
+    arrow::FieldVector struct_fields = stored_schema->fields();
+    struct_fields[3] = FieldWithId("profile", arrow::struct_({stored_profile_a, required}), 1);
+    requested_schemas.push_back(std::move(struct_fields));
+    arrow::FieldVector list_fields = stored_schema->fields();
+    list_fields[4] =
+        FieldWithId("items", arrow::list(arrow::struct_({stored_item_a, required})), 2);
+    requested_schemas.push_back(std::move(list_fields));
+    arrow::FieldVector map_fields = stored_schema->fields();
+    map_fields[5] = FieldWithId(
+        "attrs", arrow::map(arrow::utf8(), arrow::struct_({stored_attr_a, required})), 3);
+    requested_schemas.push_back(std::move(map_fields));
+
+    for (const arrow::FieldVector& fields : requested_schemas) {
+        auto c_schema = std::make_unique<ArrowSchema>();
+        ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(fields), c_schema.get()).ok());
+        RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
+                                     /*enable_predicate_pushdown=*/false};
+        ASSERT_NOK_WITH_MSG(store->CreateQueryReaders(view, /*offset_begin=*/0, context),
+                            "requested non-nullable field 'required_nested' with id 99 is absent");
+    }
+}
+
 }  // namespace
 }  // namespace paimon::test
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index bacbfa5e2..d2b2b86ce 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -19,14 +19,12 @@
 #include "paimon/core/realtime/realtime_primary_key_writer.h"
 
 #include <limits>
-#include <new>
 #include <optional>
 #include <utility>
 #include <vector>
 
 #include "arrow/api.h"
 #include "arrow/c/bridge.h"
-#include "arrow/c/helpers.h"
 #include "arrow/compute/api.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
@@ -38,6 +36,7 @@
 #include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
 #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
 #include "paimon/core/mergetree/merge_tree_writer.h"
+#include "paimon/core/realtime/arrow_array_pool_holder.h"
 #include "paimon/core/realtime/prepared_key_value_reader.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
 #include "paimon/core/utils/commit_increment.h"
@@ -47,42 +46,6 @@ namespace paimon {
 
 namespace {
 
-struct PreparedArrayPrivateData {
-    void (*release)(ArrowArray*);
-    void* private_data;
-    std::shared_ptr<arrow::MemoryPool> arrow_pool;
-};
-
-void ReleasePreparedArray(ArrowArray* array) {
-    std::unique_ptr<PreparedArrayPrivateData> data(
-        static_cast<PreparedArrayPrivateData*>(array->private_data));
-    array->release = data->release;
-    array->private_data = data->private_data;
-    array->release(array);
-}
-
-Status RetainPreparedArrayPool(ArrowArray* array,
-                               const std::shared_ptr<arrow::MemoryPool>& arrow_pool) {
-    if (!array || !array->release) {
-        return Status::Invalid("cannot retain prepared batch memory pool");
-    }
-    if (!arrow_pool) {
-        ArrowArrayRelease(array);
-        return Status::Invalid("cannot retain prepared batch memory pool");
-    }
-    std::unique_ptr<PreparedArrayPrivateData> data;
-    try {
-        data = std::make_unique<PreparedArrayPrivateData>(
-            PreparedArrayPrivateData{array->release, array->private_data, arrow_pool});
-    } catch (const std::bad_alloc&) {
-        ArrowArrayRelease(array);
-        return Status::OutOfMemory("failed to retain prepared batch memory pool");
-    }
-    array->private_data = data.release();
-    array->release = ReleasePreparedArray;
-    return Status::OK();
-}
-
 Result<std::shared_ptr<arrow::StructArray>> PrepareBatch(
     std::unique_ptr<RecordBatch>&& batch, const std::shared_ptr<arrow::Schema>& write_schema,
     const std::shared_ptr<arrow::Schema>& prepared_schema,
@@ -252,7 +215,7 @@ Status RealtimePrimaryKeyWriter::Write(std::unique_ptr<RecordBatch>&& batch) {
                      first_sequence, next_offset_, arrow_pool_.get()));
     auto output = std::make_unique<ArrowArray>();
     PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*prepared, output.get()));
-    PAIMON_RETURN_NOT_OK(RetainPreparedArrayPool(output.get(), arrow_pool_));
+    PAIMON_RETURN_NOT_OK(RetainArrowArrayMemoryPool(output.get(), arrow_pool_));
     RecordBatchBuilder builder(output.get());
     PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RecordBatch> prepared_batch, builder.Finish());
     PAIMON_RETURN_NOT_OK(realtime_store_->Write(RealtimeWriteBatch{

From ace7e1d9bbaaac1c51c2251e1c22b3dcf0773183 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Tue, 25 Aug 2026 23:57:27 +0800
Subject: [PATCH 37/62] fix(realtime): validate exact commit reader coverage

---
 .../merged_key_value_record_reader_test.cpp   | 57 +++++++++++++++++++
 .../realtime/prepared_key_value_reader.cpp    | 18 +++---
 .../realtime/primary_key_realtime_store.cpp   |  7 +++
 3 files changed, 71 insertions(+), 11 deletions(-)

diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index d86e3fc9a..bf1aaec9e 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -290,6 +290,63 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderCommitSchema) {
         "exact");
 }
 
+TEST_F(MergedKeyValueRecordReaderTest, TestCommitOffsetCoverage) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> first_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+                                                  R"([[0, 10, 2, 1], [0, 11, 0, 3]])")
+            .ValueOrDie();
+    std::shared_ptr<arrow::Array> second_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+                                                  R"([[0, 12, 1, 2], [0, 13, 3, 4]])")
+            .ValueOrDie();
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+    batch_readers.push_back(
+        std::make_unique<MockFileBatchReader>(first_array, prepared_type, /*read_batch_size=*/1));
+    batch_readers.push_back(
+        std::make_unique<MockFileBatchReader>(second_array, prepared_type, /*read_batch_size=*/1));
+
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
+                         PreparedKeyValueReaderFactory::CreateForCommit(
+                             std::move(batch_readers), prepared_schema, OffsetRange(0, 4),
+                             value_schema, value_schema, pool_));
+    int64_t row_count = 0;
+    for (const std::unique_ptr<KeyValueRecordReader>& reader : readers) {
+        ASSERT_OK_AND_ASSIGN(
+            std::vector<KeyValue> rows,
+            (ReadResultCollector::CollectKeyValueResult<
+                KeyValueRecordReader, KeyValueRecordReader::Iterator>(reader.get())));
+        row_count += static_cast<int64_t>(rows.size());
+    }
+    ASSERT_EQ(4, row_count);
+}
+
+TEST_F(MergedKeyValueRecordReaderTest, TestRejectsDuplicateCommitOffset) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(
+            prepared_type, R"([[0, 10, 0, 1], [0, 11, 0, 2], [0, 12, 2, 3]])")
+            .ValueOrDie();
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+    batch_readers.push_back(std::make_unique<MockFileBatchReader>(prepared_array, prepared_type,
+                                                                  /*read_batch_size=*/1));
+
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
+                         PreparedKeyValueReaderFactory::CreateForCommit(
+                             std::move(batch_readers), prepared_schema, OffsetRange(0, 3),
+                             value_schema, value_schema, pool_));
+    ASSERT_NOK_WITH_MSG((ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
+                                                                    KeyValueRecordReader::Iterator>(
+                            readers[0].get())),
+                        "did not cover the sealed range");
+}
+
 TEST_F(MergedKeyValueRecordReaderTest, TestBadCommitBatch) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Field> value = MakeField("value", arrow::int32(), 1);
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index d8a4cfdc0..ed1f9e158 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -18,9 +18,7 @@
 
 #include "paimon/core/realtime/prepared_key_value_reader.h"
 
-#include <algorithm>
 #include <cstdint>
-#include <limits>
 #include <memory>
 #include <optional>
 #include <utility>
@@ -44,6 +42,7 @@
 #include "paimon/macros.h"
 #include "paimon/reader/batch_reader.h"
 #include "paimon/status.h"
+#include "paimon/utils/roaring_bitmap64.h"
 
 namespace paimon {
 
@@ -81,9 +80,10 @@ class RealtimeOffsetCoverage {
                 return Status::Invalid(
                     "PK real-time store commit reader offset is outside the sealed range");
             }
-            min_seen_offset_ = std::min(min_seen_offset_, offset);
-            max_seen_offset_ = std::max(max_seen_offset_, offset);
-            ++seen_count_;
+            if (!seen_offsets_.CheckedAdd(offset)) {
+                return Status::Invalid(
+                    "PK real-time store commit readers did not cover the sealed range");
+            }
         }
         return Status::OK();
     }
@@ -91,9 +91,7 @@ class RealtimeOffsetCoverage {
     Status FinishReader() {
         ++finished_reader_count_;
         if (finished_reader_count_ == reader_count_ &&
-            (seen_count_ != sealed_offsets_.Count() ||
-             (seen_count_ > 0 && (min_seen_offset_ != sealed_offsets_.begin ||
-                                  max_seen_offset_ != sealed_offsets_.end - 1)))) {
+            seen_offsets_.Cardinality() != sealed_offsets_.Count()) {
             return Status::Invalid(
                 "PK real-time store commit readers did not cover the sealed range");
         }
@@ -106,9 +104,7 @@ class RealtimeOffsetCoverage {
 
     OffsetRange sealed_offsets_;
     size_t reader_count_;
-    int64_t min_seen_offset_ = std::numeric_limits<int64_t>::max();
-    int64_t max_seen_offset_ = std::numeric_limits<int64_t>::min();
-    int64_t seen_count_ = 0;
+    RoaringBitmap64 seen_offsets_;
     size_t finished_reader_count_ = 0;
 };
 
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index cbaef65e2..cffe1e6d0 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -26,12 +26,14 @@
 
 #include "arrow/api.h"
 #include "arrow/c/bridge.h"
+#include "arrow/c/helpers.h"
 #include "fmt/format.h"
 #include "paimon/common/metrics/metrics_impl.h"
 #include "paimon/common/utils/arrow/arrow_utils.h"
 #include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/realtime/arrow_array_pool_holder.h"
 #include "paimon/core/realtime/prepared_key_value_reader.h"
 #include "paimon/core/utils/nested_projection_utils.h"
@@ -234,12 +236,17 @@ class StoredBatchReader final : public BatchReader {
         }
         auto array = std::make_unique<ArrowArray>();
         auto schema = std::make_unique<ArrowSchema>();
+        ScopeGuard export_guard([array_ptr = array.get(), schema_ptr = schema.get()]() {
+            ArrowArrayRelease(array_ptr);
+            ArrowSchemaRelease(schema_ptr);
+        });
         PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*data_, array.get(), schema.get()));
         if (arrow_pool_) {
             PAIMON_RETURN_NOT_OK(RetainArrowArrayMemoryPool(array.get(), arrow_pool_));
         }
         data_.reset();
         arrow_pool_.reset();
+        export_guard.Release();
         return ReadBatch(std::move(array), std::move(schema));
     }
 

From 607cac7dcc30d6150a8bea61bf6ff0e8ccc317c6 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Wed, 26 Aug 2026 00:18:02 +0800
Subject: [PATCH 38/62] fix(realtime): enforce store reader boundaries

---
 .../realtime/primary_key_realtime_store.cpp   | 20 ++++---
 .../primary_key_realtime_store_test.cpp       | 54 +++++++++++++++----
 .../core/realtime/realtime_context_impl.cpp   |  6 +++
 .../core/realtime/realtime_context_test.cpp   | 23 ++++++++
 4 files changed, 86 insertions(+), 17 deletions(-)

diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index cffe1e6d0..4f6249985 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -225,7 +225,7 @@ class ReadView final : public RealtimeReadView {
 class StoredBatchReader final : public BatchReader {
  public:
     explicit StoredBatchReader(const StoredBatch& batch,
-                               std::shared_ptr<arrow::MemoryPool> arrow_pool = nullptr)
+                               std::shared_ptr<arrow::MemoryPool> arrow_pool)
         : arrow_pool_(std::move(arrow_pool)),
           data_(batch.data),
           metrics_(std::make_shared<MetricsImpl>()) {}
@@ -240,10 +240,15 @@ class StoredBatchReader final : public BatchReader {
             ArrowArrayRelease(array_ptr);
             ArrowSchemaRelease(schema_ptr);
         });
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*data_, array.get(), schema.get()));
-        if (arrow_pool_) {
-            PAIMON_RETURN_NOT_OK(RetainArrowArrayMemoryPool(array.get(), arrow_pool_));
-        }
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+            std::shared_ptr<arrow::RecordBatch> record_batch,
+            arrow::RecordBatch::FromStructArray(data_, arrow_pool_.get()));
+        PAIMON_ASSIGN_OR_RAISE(
+            std::shared_ptr<arrow::RecordBatch> normalized_batch,
+            ArrowUtils::NormalizeRecordBatchOffsets(record_batch, arrow_pool_.get()));
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(
+            arrow::ExportRecordBatch(*normalized_batch, array.get(), schema.get()));
+        PAIMON_RETURN_NOT_OK(RetainArrowArrayMemoryPool(array.get(), arrow_pool_));
         data_.reset();
         arrow_pool_.reset();
         export_guard.Release();
@@ -321,7 +326,7 @@ class PrimaryKeyRealtimeStore::Impl {
         std::vector<std::unique_ptr<BatchReader>> readers;
         readers.reserve(segment->Batches().size());
         for (const StoredBatch& batch : segment->Batches()) {
-            readers.push_back(std::make_unique<StoredBatchReader>(batch));
+            readers.push_back(std::make_unique<StoredBatchReader>(batch, arrow_pool_));
         }
         return readers;
     }
@@ -360,8 +365,7 @@ class PrimaryKeyRealtimeStore::Impl {
                                           arrow_pool_.get()));
                 StoredBatch query_batch{checked_pointer_cast<arrow::StructArray>(projected.array),
                                         batch.offset_range, /*memory_usage=*/0};
-                readers.push_back(std::make_unique<StoredBatchReader>(
-                    query_batch, projected.uses_arrow_pool ? arrow_pool_ : nullptr));
+                readers.push_back(std::make_unique<StoredBatchReader>(query_batch, arrow_pool_));
             }
         }
         return readers;
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index a63d79eec..7b2e09245 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -255,12 +255,45 @@ TEST(PrimaryKeyRealtimeStoreTest, TestCommitReaderPerStoredBatch) {
     }
 }
 
-TEST(PrimaryKeyRealtimeStoreTest, TestCommitReaderExportsZeroOffsets) {
+void AssertSlicedBatch(BatchReader* reader) {
+    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch());
+    ASSERT_FALSE(BatchReader::IsEofBatch(batch));
+    ASSERT_EQ(2, batch.first->length);
+    AssertOffsetsZero(batch.first.get());
+    arrow::Result<std::shared_ptr<arrow::Array>> import_result =
+        arrow::ImportArray(batch.first.get(), batch.second.get());
+    ASSERT_TRUE(import_result.ok()) << import_result.status().ToString();
+    std::shared_ptr<arrow::Array> array = std::move(import_result).ValueOrDie();
+    std::shared_ptr<arrow::StructArray> values = checked_pointer_cast<arrow::StructArray>(array);
+    ASSERT_EQ(2, checked_pointer_cast<arrow::Int64Array>(values->field(3))->Value(0));
+    ASSERT_EQ(3, checked_pointer_cast<arrow::Int64Array>(values->field(3))->Value(1));
+    std::shared_ptr<arrow::StructArray> nested =
+        checked_pointer_cast<arrow::StructArray>(values->field(4));
+    ASSERT_EQ("two", checked_pointer_cast<arrow::StringArray>(nested->field(0))->GetString(0));
+    ASSERT_EQ("three", checked_pointer_cast<arrow::StringArray>(nested->field(0))->GetString(1));
+    std::shared_ptr<arrow::ListArray> items =
+        checked_pointer_cast<arrow::ListArray>(nested->field(1));
+    std::shared_ptr<arrow::Int32Array> first_items =
+        checked_pointer_cast<arrow::Int32Array>(items->value_slice(0));
+    ASSERT_EQ(3, first_items->Value(0));
+    ASSERT_EQ(4, first_items->Value(1));
+    std::shared_ptr<arrow::Int32Array> second_items =
+        checked_pointer_cast<arrow::Int32Array>(items->value_slice(1));
+    ASSERT_EQ(5, second_items->Value(0));
+    ASSERT_EQ(6, second_items->Value(1));
+    ASSERT_OK_AND_ASSIGN(batch, reader->NextBatch());
+    ASSERT_TRUE(BatchReader::IsEofBatch(batch));
+}
+
+TEST(PrimaryKeyRealtimeStoreTest, TestSlicedReadersExportZeroOffsets) {
     std::shared_ptr<arrow::Schema> schema = NestedPreparedSchema();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
                          PrimaryKeyRealtimeStore::Create(schema, GetDefaultPool()));
     ASSERT_OK(store->Write(RealtimeWriteBatch{
-        MakeBatch(schema, R"([[0, 1, 0, 1, ["one", [1, 2]]], [0, 2, 1, 2, ["two", [3, 4]]]])"),
+        MakeSlicedBatch(
+            schema,
+            R"([[0, 1, 0, 1, ["one", [1, 2]]], [0, 2, 1, 2, ["two", [3, 4]]], [0, 3, 2, 3, ["three", [5, 6]]], [0, 4, 3, 4, ["four", [7, 8]]]])",
+            1, 2),
         OffsetRange(0, 2)}));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
                          store->SealForCommit());
@@ -268,13 +301,16 @@ TEST(PrimaryKeyRealtimeStoreTest, TestCommitReaderExportsZeroOffsets) {
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
                          store->CreateCommitReaders(segment.value()));
     ASSERT_EQ(1, readers.size());
-    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, readers[0]->NextBatch());
-    ASSERT_FALSE(BatchReader::IsEofBatch(batch));
-    ASSERT_EQ(2, batch.first->length);
-    AssertOffsetsZero(batch.first.get());
-    ASSERT_TRUE(arrow::ImportArray(batch.first.get(), batch.second.get()).ok());
-    ASSERT_OK_AND_ASSIGN(batch, readers[0]->NextBatch());
-    ASSERT_TRUE(BatchReader::IsEofBatch(batch));
+    AssertSlicedBatch(readers[0].get());
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
+    auto c_schema = std::make_unique<ArrowSchema>();
+    ASSERT_TRUE(arrow::ExportSchema(*schema, c_schema.get()).ok());
+    RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+    ASSERT_OK_AND_ASSIGN(readers, store->CreateQueryReaders(view, /*offset_begin=*/0, context));
+    ASSERT_EQ(1, readers.size());
+    AssertSlicedBatch(readers[0].get());
 }
 
 TEST(PrimaryKeyRealtimeStoreTest, TestCloseUnreadBatchReaders) {
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index 4b5da718e..ad17ee695 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -150,6 +150,9 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     RealtimeStoreMode mode = request.mode;
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> store,
                            factory_->Create(std::move(request)));
+    if (!store) {
+        return Status::Invalid("real-time store factory returned a null store");
+    }
     stores_.emplace(partition_bucket, StoreEntry{store, requested_schema, mode});
     if (offset_iter != committed_offsets_.end()) {
         reclaimed_offsets_.emplace(partition_bucket, offset_iter->second);
@@ -180,6 +183,9 @@ Result<std::vector<RealtimePartitionBucketView>> RealtimeContextImpl::AcquireRea
     for (const auto& [partition_bucket, store] : stores_) {
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeReadView> read_view,
                                store.store->AcquireReadView());
+        if (!read_view) {
+            return Status::Invalid("real-time store returned a null read view");
+        }
         result.push_back(
             RealtimePartitionBucketView{partition_bucket, store.store, std::move(read_view)});
     }
diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp
index a418c17a8..6dd1d7577 100644
--- a/src/paimon/core/realtime/realtime_context_test.cpp
+++ b/src/paimon/core/realtime/realtime_context_test.cpp
@@ -56,6 +56,9 @@ class TestingRealtimeStore : public RealtimeStore {
     }
     Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
         ++acquire_count;
+        if (return_null_read_view) {
+            return std::shared_ptr<RealtimeReadView>();
+        }
         return std::make_shared<TestingReadView>();
     }
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
@@ -78,6 +81,7 @@ class TestingRealtimeStore : public RealtimeStore {
     int32_t acquire_count = 0;
     int32_t advance_count = 0;
     bool fail_next_advance = false;
+    bool return_null_read_view = false;
     std::vector<int64_t> committed_offsets;
 };
 
@@ -88,11 +92,15 @@ class TestingRealtimeStoreFactory : public RealtimeStoreFactory {
             return Status::Invalid("testing write schema is null");
         }
         ArrowSchemaRelease(request.write_schema.get());
+        if (return_null_store) {
+            return std::shared_ptr<RealtimeStore>();
+        }
         auto store = std::make_shared<TestingRealtimeStore>();
         stores.push_back(store);
         return store;
     }
 
+    bool return_null_store = false;
     std::vector<std::shared_ptr<TestingRealtimeStore>> stores;
 };
 
@@ -421,5 +429,20 @@ TEST(RealtimeContextTest, TestRejectsNullFactory) {
                         "real-time store factory is null");
 }
 
+TEST(RealtimeContextTest, TestRejectsNullPluginResults) {
+    auto factory = std::make_shared<TestingRealtimeStoreFactory>();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
+    factory->return_null_store = true;
+    ASSERT_NOK_WITH_MSG(GetOrCreateAppendStore(context, /*partition=*/{}, /*bucket=*/0,
+                                               MakeWriteSchema(), {}, GetDefaultPool()),
+                        "real-time store factory returned a null store");
+
+    factory->return_null_store = false;
+    ASSERT_OK(GetOrCreateAppendStore(context, /*partition=*/{}, /*bucket=*/0, MakeWriteSchema(), {},
+                                     GetDefaultPool()));
+    factory->stores[0]->return_null_read_view = true;
+    ASSERT_NOK_WITH_MSG(context->AcquireReadViews(), "real-time store returned a null read view");
+}
+
 }  // namespace
 }  // namespace paimon::test

From 86123e0b54d57d92c50ca6e7bf5990f79be436b7 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Wed, 26 Aug 2026 11:22:23 +0800
Subject: [PATCH 39/62] refactor(realtime): align primary key query projection

---
 src/paimon/CMakeLists.txt                     |   2 -
 .../core/operation/file_store_write.cpp       |   3 +-
 .../primary_key_realtime_validator.cpp        |  80 -------
 .../primary_key_realtime_validator.h          |  36 ---
 .../primary_key_realtime_validator_test.cpp   |  91 --------
 .../realtime/primary_key_realtime_store.cpp   | 147 +-----------
 .../primary_key_realtime_store_test.cpp       | 217 +-----------------
 src/paimon/core/table/source/table_scan.cpp   |   4 +-
 .../core/utils/primary_key_table_utils.cpp    |  50 ++++
 .../core/utils/primary_key_table_utils.h      |   3 +
 .../utils/primary_key_table_utils_test.cpp    |  58 +++++
 test/inte/realtime_write_inte_test.cpp        |  39 ----
 12 files changed, 132 insertions(+), 598 deletions(-)
 delete mode 100644 src/paimon/core/realtime/framework/primary_key_realtime_validator.cpp
 delete mode 100644 src/paimon/core/realtime/framework/primary_key_realtime_validator.h
 delete mode 100644 src/paimon/core/realtime/framework/primary_key_realtime_validator_test.cpp

diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index fb702bfea..55d033570 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -383,7 +383,6 @@ set(PAIMON_CORE_SRCS
     core/realtime/arrow_array_pool_holder.cpp
     core/realtime/arrow_realtime_store.cpp
     core/realtime/arrow_realtime_store_factory.cpp
-    core/realtime/framework/primary_key_realtime_validator.cpp
     core/realtime/prepared_key_value_reader.cpp
     core/realtime/primary_key_realtime_store.cpp
     core/realtime/realtime_append_only_writer.cpp
@@ -792,7 +791,6 @@ if(PAIMON_BUILD_TESTS)
                     core/manifest/index_manifest_file_handler_test.cpp
                     core/memory/writer_memory_manager_test.cpp
                     core/realtime/arrow_realtime_store_test.cpp
-                    core/realtime/framework/primary_key_realtime_validator_test.cpp
                     core/realtime/primary_key_realtime_store_test.cpp
                     core/realtime/realtime_context_test.cpp
                     core/realtime/realtime_reader_test.cpp
diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp
index f77dadda2..9710b57f0 100644
--- a/src/paimon/core/operation/file_store_write.cpp
+++ b/src/paimon/core/operation/file_store_write.cpp
@@ -36,7 +36,6 @@
 #include "paimon/core/operation/key_value_file_store_write.h"
 #include "paimon/core/options/merge_engine.h"
 #include "paimon/core/postpone/postpone_bucket_file_store_write.h"
-#include "paimon/core/realtime/framework/primary_key_realtime_validator.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
 #include "paimon/core/schema/schema_manager.h"
 #include "paimon/core/schema/table_schema.h"
@@ -210,7 +209,7 @@ Result<std::unique_ptr<FileStoreWrite>> FileStoreWrite::Create(std::unique_ptr<W
     } else {
         // pk table
         if (ctx->GetRealtimeContext()) {
-            PAIMON_RETURN_NOT_OK(PrimaryKeyRealtimeValidator::ValidateOptions(options, *schema));
+            PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *schema));
             if (ignore_previous_files) {
                 return Status::NotImplemented(
                     "PK realtime v1 requires restore from the latest snapshot");
diff --git a/src/paimon/core/realtime/framework/primary_key_realtime_validator.cpp b/src/paimon/core/realtime/framework/primary_key_realtime_validator.cpp
deleted file mode 100644
index 61c640ee6..000000000
--- a/src/paimon/core/realtime/framework/primary_key_realtime_validator.cpp
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "paimon/core/realtime/framework/primary_key_realtime_validator.h"
-
-#include <vector>
-
-#include "arrow/type.h"
-#include "paimon/common/types/data_field.h"
-#include "paimon/core/core_options.h"
-#include "paimon/core/index/pk/primary_key_index_definitions.h"
-#include "paimon/core/schema/table_schema.h"
-#include "paimon/macros.h"
-
-namespace paimon {
-
-Status PrimaryKeyRealtimeValidator::ValidateOptions(const CoreOptions& options,
-                                                    const TableSchema& schema) {
-    if (options.GetBucket() <= 0) {
-        return Status::NotImplemented("PK realtime v1 requires fixed buckets");
-    }
-    if (options.GetMergeEngine() != MergeEngine::DEDUPLICATE) {
-        return Status::NotImplemented("PK realtime v1 supports only the DEDUPLICATE merge engine");
-    }
-    if (options.DataEvolutionEnabled()) {
-        return Status::NotImplemented("PK realtime v1 does not support data evolution");
-    }
-    if (!options.GetFieldsSequenceGroups().empty()) {
-        return Status::NotImplemented("PK realtime v1 does not support sequence groups");
-    }
-    if (options.IgnoreDelete() || options.PartialUpdateRemoveRecordOnDelete() ||
-        options.AggregationRemoveRecordOnDelete() ||
-        !options.GetPartialUpdateRemoveRecordOnSequenceGroup().empty()) {
-        return Status::NotImplemented("PK realtime v1 requires default delete behavior");
-    }
-    if (!options.GetSequenceField().empty()) {
-        return Status::NotImplemented("PK realtime v1 does not support sequence.field");
-    }
-    if (!options.SequenceFieldSortOrderIsAscending()) {
-        return Status::NotImplemented(
-            "PK realtime v1 supports only ascending sequence.field.sort-order");
-    }
-    if (options.NeedLookup() || options.DeletionVectorsEnabled() ||
-        options.GetChangelogProducer() != ChangelogProducer::NONE) {
-        return Status::NotImplemented("PK realtime v1 does not support lookup or early MOR");
-    }
-    PAIMON_ASSIGN_OR_RAISE(std::vector<DataField> primary_key_fields,
-                           schema.TrimmedPrimaryKeyFields());
-    for (const DataField& field : primary_key_fields) {
-        if (field.Type()->id() == arrow::Type::FLOAT || field.Type()->id() == arrow::Type::DOUBLE) {
-            return Status::NotImplemented(
-                "PK realtime v1 does not support FLOAT or DOUBLE primary keys");
-        }
-    }
-    if (options.GlobalIndexEnabled()) {
-        PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions definitions,
-                               PrimaryKeyIndexDefinitions::Create(schema));
-        if (!definitions.Definitions().empty()) {
-            return Status::NotImplemented("PK realtime v1 does not support global indexes");
-        }
-    }
-    return Status::OK();
-}
-
-}  // namespace paimon
diff --git a/src/paimon/core/realtime/framework/primary_key_realtime_validator.h b/src/paimon/core/realtime/framework/primary_key_realtime_validator.h
deleted file mode 100644
index fa9432d3b..000000000
--- a/src/paimon/core/realtime/framework/primary_key_realtime_validator.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-
-#include "paimon/status.h"
-
-namespace paimon {
-
-class CoreOptions;
-class TableSchema;
-
-class PrimaryKeyRealtimeValidator {
- public:
-    PrimaryKeyRealtimeValidator() = delete;
-    ~PrimaryKeyRealtimeValidator() = delete;
-
-    static Status ValidateOptions(const CoreOptions& options, const TableSchema& schema);
-};
-
-}  // namespace paimon
diff --git a/src/paimon/core/realtime/framework/primary_key_realtime_validator_test.cpp b/src/paimon/core/realtime/framework/primary_key_realtime_validator_test.cpp
deleted file mode 100644
index 3b7793b14..000000000
--- a/src/paimon/core/realtime/framework/primary_key_realtime_validator_test.cpp
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "paimon/core/realtime/framework/primary_key_realtime_validator.h"
-
-#include <map>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "arrow/type.h"
-#include "gtest/gtest.h"
-#include "paimon/core/core_options.h"
-#include "paimon/core/schema/table_schema.h"
-#include "paimon/defs.h"
-#include "paimon/testing/utils/testharness.h"
-
-namespace paimon::test {
-namespace {
-
-std::shared_ptr<TableSchema> PkSchema(
-    const std::shared_ptr<arrow::DataType>& key_type = arrow::int64(),
-    const std::map<std::string, std::string>& options = {}) {
-    return TableSchema::Create(
-               /*schema_id=*/0,
-               arrow::schema({arrow::field("id", key_type), arrow::field("value", arrow::utf8())}),
-               /*partition_keys=*/{}, /*primary_keys=*/{"id"}, options)
-        .value();
-}
-
-}  // namespace
-
-TEST(PrimaryKeyRealtimeValidatorTest, TestSupportedOptions) {
-    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
-    ASSERT_OK(PrimaryKeyRealtimeValidator::ValidateOptions(options, *PkSchema()));
-}
-
-TEST(PrimaryKeyRealtimeValidatorTest, TestUnsupportedOptions) {
-    const std::string sequence_group =
-        std::string(Options::FIELDS_PREFIX) + ".value." + Options::SEQUENCE_GROUP;
-    const std::vector<std::map<std::string, std::string>> unsupported_options = {
-        {{Options::BUCKET, "0"}},
-        {{Options::BUCKET, "1"}, {Options::MERGE_ENGINE, "partial-update"}},
-        {{Options::BUCKET, "1"}, {Options::DATA_EVOLUTION_ENABLED, "true"}},
-        {{Options::BUCKET, "1"}, {sequence_group, "seq"}},
-        {{Options::BUCKET, "1"}, {Options::SEQUENCE_FIELD, "seq"}},
-        {{Options::BUCKET, "1"}, {Options::FORCE_LOOKUP, "true"}},
-        {{Options::BUCKET, "1"}, {Options::DELETION_VECTORS_ENABLED, "true"}},
-        {{Options::BUCKET, "1"}, {Options::CHANGELOG_PRODUCER, "input"}},
-    };
-    for (const std::map<std::string, std::string>& option_map : unsupported_options) {
-        ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
-        ASSERT_NOK(PrimaryKeyRealtimeValidator::ValidateOptions(options, *PkSchema()));
-    }
-}
-
-TEST(PrimaryKeyRealtimeValidatorTest, TestRejectsFloatingPrimaryKeys) {
-    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
-    ASSERT_NOK_WITH_MSG(
-        PrimaryKeyRealtimeValidator::ValidateOptions(options, *PkSchema(arrow::float32())),
-        "FLOAT or DOUBLE primary keys");
-    ASSERT_NOK_WITH_MSG(
-        PrimaryKeyRealtimeValidator::ValidateOptions(options, *PkSchema(arrow::float64())),
-        "FLOAT or DOUBLE primary keys");
-}
-
-TEST(PrimaryKeyRealtimeValidatorTest, TestRejectsEnabledGlobalIndex) {
-    const std::map<std::string, std::string> option_map = {{Options::BUCKET, "1"},
-                                                           {Options::PK_BTREE_INDEX_COLUMNS, "id"}};
-    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
-    ASSERT_NOK_WITH_MSG(PrimaryKeyRealtimeValidator::ValidateOptions(
-                            options, *PkSchema(arrow::int64(), option_map)),
-                        "does not support global indexes");
-}
-
-}  // namespace paimon::test
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index 4f6249985..4e1defdf5 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -20,14 +20,12 @@
 
 #include <mutex>
 #include <optional>
-#include <unordered_map>
 #include <utility>
 #include <vector>
 
 #include "arrow/api.h"
 #include "arrow/c/bridge.h"
 #include "arrow/c/helpers.h"
-#include "fmt/format.h"
 #include "paimon/common/metrics/metrics_impl.h"
 #include "paimon/common/utils/arrow/arrow_utils.h"
 #include "paimon/common/utils/arrow/mem_utils.h"
@@ -44,139 +42,6 @@ namespace paimon {
 
 namespace {
 
-struct AlignedArray {
-    std::shared_ptr<arrow::Array> array;
-    bool uses_arrow_pool;
-};
-
-Result<AlignedArray> AlignArrayByPaimonIds(const std::shared_ptr<arrow::Array>& array,
-                                           const std::shared_ptr<arrow::DataType>& read_type,
-                                           arrow::MemoryPool* pool);
-
-bool TypesExactlyEqual(const std::shared_ptr<arrow::DataType>& data_type,
-                       const std::shared_ptr<arrow::DataType>& read_type) {
-    if (!data_type->Equals(read_type) || data_type->num_fields() != read_type->num_fields()) {
-        return false;
-    }
-    for (int32_t i = 0; i < data_type->num_fields(); ++i) {
-        if (!data_type->field(i)->Equals(read_type->field(i), /*check_metadata=*/true) ||
-            !TypesExactlyEqual(data_type->field(i)->type(), read_type->field(i)->type())) {
-            return false;
-        }
-    }
-    return true;
-}
-
-Result<AlignedArray> AlignStructArrayByPaimonIds(
-    const std::shared_ptr<arrow::StructArray>& array,
-    const std::shared_ptr<arrow::StructType>& read_type, arrow::MemoryPool* pool) {
-    const std::shared_ptr<arrow::StructType> data_type =
-        checked_pointer_cast<arrow::StructType>(array->type());
-    std::unordered_map<int32_t, int32_t> data_field_indexes;
-    data_field_indexes.reserve(data_type->num_fields());
-    for (int32_t i = 0; i < data_type->num_fields(); ++i) {
-        PAIMON_ASSIGN_OR_RAISE(int32_t field_id,
-                               NestedProjectionUtils::GetPaimonFieldId(data_type->field(i)));
-        if (!data_field_indexes.emplace(field_id, i).second) {
-            return Status::Invalid(fmt::format("duplicate field id {} in stored schema", field_id));
-        }
-    }
-
-    std::unordered_map<int32_t, bool> requested_field_ids;
-    requested_field_ids.reserve(read_type->num_fields());
-    std::vector<std::shared_ptr<arrow::ArrayData>> children;
-    children.reserve(read_type->num_fields());
-    bool uses_arrow_pool = false;
-    for (const std::shared_ptr<arrow::Field>& read_field : read_type->fields()) {
-        PAIMON_ASSIGN_OR_RAISE(int32_t field_id,
-                               NestedProjectionUtils::GetPaimonFieldId(read_field));
-        if (!requested_field_ids.emplace(field_id, true).second) {
-            return Status::Invalid(
-                fmt::format("duplicate field id {} in requested schema", field_id));
-        }
-        const auto data_iter = data_field_indexes.find(field_id);
-        if (data_iter == data_field_indexes.end()) {
-            if (!read_field->nullable()) {
-                return Status::Invalid(fmt::format(
-                    "requested non-nullable field '{}' with id {} is absent from stored schema",
-                    read_field->name(), field_id));
-            }
-            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-                std::shared_ptr<arrow::Array> null_child,
-                arrow::MakeArrayOfNull(read_field->type(), array->offset() + array->length(),
-                                       pool));
-            children.push_back(null_child->data());
-            uses_arrow_pool = true;
-            continue;
-        }
-        std::shared_ptr<arrow::Array> child =
-            arrow::MakeArray(array->data()->child_data[data_iter->second]);
-        PAIMON_ASSIGN_OR_RAISE(AlignedArray aligned_child,
-                               AlignArrayByPaimonIds(child, read_field->type(), pool));
-        children.push_back(aligned_child.array->data());
-        uses_arrow_pool = uses_arrow_pool || aligned_child.uses_arrow_pool;
-    }
-
-    std::shared_ptr<arrow::ArrayData> aligned = array->data()->Copy();
-    aligned->type = read_type;
-    aligned->child_data = std::move(children);
-    return AlignedArray{arrow::MakeArray(std::move(aligned)), uses_arrow_pool};
-}
-
-Result<AlignedArray> AlignArrayByPaimonIds(const std::shared_ptr<arrow::Array>& array,
-                                           const std::shared_ptr<arrow::DataType>& read_type,
-                                           arrow::MemoryPool* pool) {
-    if (TypesExactlyEqual(array->type(), read_type)) {
-        return AlignedArray{array, false};
-    }
-    if (array->type_id() != read_type->id()) {
-        return Status::Invalid(fmt::format("stored value type {} does not match requested type {}",
-                                           array->type()->ToString(), read_type->ToString()));
-    }
-    switch (read_type->id()) {
-        case arrow::Type::STRUCT:
-            return AlignStructArrayByPaimonIds(checked_pointer_cast<arrow::StructArray>(array),
-                                               checked_pointer_cast<arrow::StructType>(read_type),
-                                               pool);
-        case arrow::Type::LIST: {
-            std::shared_ptr<arrow::Array> values =
-                checked_pointer_cast<arrow::ListArray>(array)->values();
-            PAIMON_ASSIGN_OR_RAISE(
-                AlignedArray aligned_values,
-                AlignArrayByPaimonIds(values, read_type->field(0)->type(), pool));
-            std::shared_ptr<arrow::ArrayData> aligned = array->data()->Copy();
-            aligned->type = read_type;
-            aligned->child_data = {aligned_values.array->data()};
-            return AlignedArray{arrow::MakeArray(std::move(aligned)),
-                                aligned_values.uses_arrow_pool};
-        }
-        case arrow::Type::MAP: {
-            const std::shared_ptr<arrow::MapArray> map =
-                checked_pointer_cast<arrow::MapArray>(array);
-            const std::shared_ptr<arrow::MapType> map_type =
-                checked_pointer_cast<arrow::MapType>(read_type);
-            std::shared_ptr<arrow::Array> keys = map->keys();
-            PAIMON_ASSIGN_OR_RAISE(AlignedArray aligned_keys,
-                                   AlignArrayByPaimonIds(keys, map_type->key_type(), pool));
-            std::shared_ptr<arrow::Array> items = map->items();
-            PAIMON_ASSIGN_OR_RAISE(AlignedArray aligned_items,
-                                   AlignArrayByPaimonIds(items, map_type->item_type(), pool));
-            std::shared_ptr<arrow::ArrayData> entries = array->data()->child_data[0]->Copy();
-            entries->type = arrow::struct_({map_type->key_field(), map_type->item_field()});
-            entries->child_data = {aligned_keys.array->data(), aligned_items.array->data()};
-            std::shared_ptr<arrow::ArrayData> aligned = array->data()->Copy();
-            aligned->type = read_type;
-            aligned->child_data = {std::move(entries)};
-            return AlignedArray{arrow::MakeArray(std::move(aligned)),
-                                aligned_keys.uses_arrow_pool || aligned_items.uses_arrow_pool};
-        }
-        default:
-            return Status::Invalid(
-                fmt::format("stored leaf type {} does not match requested type {}",
-                            array->type()->ToString(), read_type->ToString()));
-    }
-}
-
 struct StoredBatch {
     std::shared_ptr<arrow::StructArray> data;
     OffsetRange offset_range;
@@ -360,10 +225,14 @@ class PrimaryKeyRealtimeStore::Impl {
         for (const std::shared_ptr<Segment>& segment : typed->Segments()) {
             for (const StoredBatch& batch : segment->Batches()) {
                 PAIMON_ASSIGN_OR_RAISE(
-                    AlignedArray projected,
-                    AlignArrayByPaimonIds(batch.data, arrow::struct_(read_schema->fields()),
-                                          arrow_pool_.get()));
-                StoredBatch query_batch{checked_pointer_cast<arrow::StructArray>(projected.array),
+                    std::shared_ptr<arrow::Array> projected,
+                    NestedProjectionUtils::AlignArrayToReadType(
+                        batch.data, arrow::struct_(read_schema->fields()), arrow_pool_.get()));
+                if (!projected || projected->type_id() != arrow::Type::STRUCT) {
+                    return Status::Invalid(
+                        "PK memory query projection did not produce a StructArray");
+                }
+                StoredBatch query_batch{checked_pointer_cast<arrow::StructArray>(projected),
                                         batch.offset_range, /*memory_usage=*/0};
                 readers.push_back(std::make_unique<StoredBatchReader>(query_batch, arrow_pool_));
             }
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index 7b2e09245..5f2e46928 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -21,7 +21,6 @@
 #include <cstddef>
 #include <cstdint>
 #include <memory>
-#include <new>
 #include <optional>
 #include <string>
 #include <utility>
@@ -84,16 +83,6 @@ std::unique_ptr<RecordBatch> MakeBatch(const std::string& json) {
     return RecordBatchBuilder(c_array.get()).Finish().value();
 }
 
-std::unique_ptr<RecordBatch> MakeBatch(const std::shared_ptr<arrow::Schema>& schema,
-                                       const std::string& json) {
-    std::shared_ptr<arrow::Array> array =
-        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema->fields()), json)
-            .ValueOrDie();
-    auto c_array = std::make_unique<ArrowArray>();
-    EXPECT_TRUE(arrow::ExportArray(*array, c_array.get()).ok());
-    return RecordBatchBuilder(c_array.get()).Finish().value();
-}
-
 std::unique_ptr<RecordBatch> MakeSlicedBatch(const std::shared_ptr<arrow::Schema>& schema,
                                              const std::string& json, int64_t offset,
                                              int64_t length) {
@@ -139,18 +128,10 @@ Result<std::string> ReadJson(const std::vector<std::unique_ptr<BatchReader>>& re
 class TestingMemoryPool final : public MemoryPool {
  public:
     void* Malloc(uint64_t size, uint64_t alignment) override {
-        ++allocation_count;
-        if (reject_allocations) {
-            throw std::bad_alloc();
-        }
         return delegate_->Malloc(size, alignment);
     }
 
     void* Realloc(void* pointer, size_t old_size, size_t new_size, uint64_t alignment) override {
-        ++allocation_count;
-        if (reject_allocations) {
-            throw std::bad_alloc();
-        }
         return delegate_->Realloc(pointer, old_size, new_size, alignment);
     }
 
@@ -170,9 +151,6 @@ class TestingMemoryPool final : public MemoryPool {
         return delegate_->MaxMemoryUsage();
     }
 
-    bool reject_allocations = false;
-    int64_t allocation_count = 0;
-
  private:
     std::unique_ptr<MemoryPool> delegate_ = GetMemoryPool();
 };
@@ -368,111 +346,7 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderPerStoredBatch) {
     ASSERT_NE(std::string::npos, actual.find("\"two\""));
 }
 
-TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderProjectsTopLevelFieldsById) {
-    const std::shared_ptr<arrow::Schema> stored_schema = PreparedSchema();
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(stored_schema, GetDefaultPool()));
-    ASSERT_OK(store->Write(RealtimeWriteBatch{
-        MakeSlicedBatch(stored_schema,
-                        R"([[0, 1, 0, 6, "six"], [0, 2, 1, 7, "seven"], [0, 3, 2, 8, "eight"]])", 1,
-                        1),
-        OffsetRange(0, 1)}));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
-
-    arrow::FieldVector requested_fields(stored_schema->fields().begin(),
-                                        stored_schema->fields().begin() + 3);
-    requested_fields.push_back(FieldWithId("renamed_value", arrow::utf8(), 1));
-    requested_fields.push_back(FieldWithId("added", arrow::int32(), 2));
-    std::shared_ptr<arrow::Schema> requested_schema = arrow::schema(std::move(requested_fields));
-    auto c_schema = std::make_unique<ArrowSchema>();
-    ASSERT_TRUE(arrow::ExportSchema(*requested_schema, c_schema.get()).ok());
-    RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
-                                 /*enable_predicate_pushdown=*/false};
-    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
-                         store->CreateQueryReaders(view, /*offset_begin=*/0, context));
-    ASSERT_EQ(1, readers.size());
-    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, readers[0]->NextBatch());
-    arrow::Result<std::shared_ptr<arrow::Array>> import_result =
-        arrow::ImportArray(batch.first.get(), batch.second.get());
-    ASSERT_TRUE(import_result.ok()) << import_result.status().ToString();
-    std::shared_ptr<arrow::Array> array = std::move(import_result).ValueOrDie();
-    ASSERT_TRUE(array->type()->Equals(arrow::struct_(requested_schema->fields())));
-    std::shared_ptr<arrow::StructArray> projected = checked_pointer_cast<arrow::StructArray>(array);
-    ASSERT_EQ(5, projected->num_fields());
-    ASSERT_EQ("seven", checked_pointer_cast<arrow::StringArray>(projected->field(3))->GetString(0));
-    ASSERT_TRUE(projected->field(4)->IsNull(0));
-}
-
-TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderRejectsMissingNonNullableTopLevelField) {
-    const std::shared_ptr<arrow::Schema> stored_schema = PreparedSchema();
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(stored_schema, GetDefaultPool()));
-    ASSERT_OK(store->Write(
-        RealtimeWriteBatch{MakeBatch(R"([[0, 1, 0, 7, "seven"]])"), OffsetRange(0, 1)}));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
-
-    arrow::FieldVector requested_fields = stored_schema->fields();
-    requested_fields.push_back(
-        FieldWithId("required_added", arrow::int32(), 2, /*nullable=*/false));
-    auto c_schema = std::make_unique<ArrowSchema>();
-    ASSERT_TRUE(
-        arrow::ExportSchema(*arrow::schema(std::move(requested_fields)), c_schema.get()).ok());
-    RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
-                                 /*enable_predicate_pushdown=*/false};
-    ASSERT_NOK_WITH_MSG(store->CreateQueryReaders(view, /*offset_begin=*/0, context),
-                        "requested non-nullable field 'required_added' with id 2 is absent");
-}
-
-TEST(PrimaryKeyRealtimeStoreTest, TestQuerySchemaAlignmentUsesCallerPool) {
-    const std::shared_ptr<arrow::Schema> stored_schema = PreparedSchema();
-    std::shared_ptr<TestingMemoryPool> pool = std::make_shared<TestingMemoryPool>();
-    auto write_schema = std::make_unique<ArrowSchema>();
-    ASSERT_TRUE(arrow::ExportSchema(*stored_schema, write_schema.get()).ok());
-    RealtimeStoreCreateRequest request{std::move(write_schema),
-                                       /*options=*/{}, pool, RealtimeStoreMode::PRIMARY_KEY};
-    ArrowRealtimeStoreFactory factory;
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeStore> store, factory.Create(std::move(request)));
-    ASSERT_OK(store->Write(
-        RealtimeWriteBatch{MakeBatch(R"([[0, 1, 0, 7, "seven"]])"), OffsetRange(0, 1)}));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
-
-    arrow::FieldVector requested_fields = stored_schema->fields();
-    requested_fields.push_back(FieldWithId("added", arrow::int32(), 2));
-    std::shared_ptr<arrow::Schema> requested_schema = arrow::schema(std::move(requested_fields));
-    auto c_schema = std::make_unique<ArrowSchema>();
-    ASSERT_TRUE(arrow::ExportSchema(*requested_schema, c_schema.get()).ok());
-    RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
-                                 /*enable_predicate_pushdown=*/false};
-
-    std::vector<arrow::FieldVector> zero_copy_schemas;
-    zero_copy_schemas.push_back(stored_schema->fields());
-    arrow::FieldVector reordered_fields(stored_schema->fields().begin(),
-                                        stored_schema->fields().begin() + 3);
-    reordered_fields.push_back(FieldWithId("renamed_value", arrow::utf8(), 1));
-    reordered_fields.push_back(FieldWithId("renamed_id", arrow::int64(), 0));
-    zero_copy_schemas.push_back(std::move(reordered_fields));
-    pool->reject_allocations = true;
-    for (const arrow::FieldVector& fields : zero_copy_schemas) {
-        auto zero_copy_schema = std::make_unique<ArrowSchema>();
-        ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(fields), zero_copy_schema.get()).ok());
-        RealtimeQueryContext zero_copy_context{zero_copy_schema.get(), /*predicate=*/nullptr,
-                                               /*enable_predicate_pushdown=*/false};
-        const int64_t allocations_before_query = pool->allocation_count;
-        ASSERT_OK_AND_ASSIGN(
-            std::vector<std::unique_ptr<BatchReader>> readers,
-            store->CreateQueryReaders(view, /*offset_begin=*/0, zero_copy_context));
-        ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, readers[0]->NextBatch());
-        ASSERT_TRUE(arrow::ImportArray(batch.first.get(), batch.second.get()).ok());
-        ASSERT_EQ(allocations_before_query, pool->allocation_count);
-    }
-
-    const int64_t allocations_before_query = pool->allocation_count;
-    ASSERT_NOK_WITH_MSG(store->CreateQueryReaders(view, /*offset_begin=*/0, context),
-                        "Out of memory");
-    ASSERT_GT(pool->allocation_count, allocations_before_query);
-}
-
-TEST(PrimaryKeyRealtimeStoreTest, TestQueryAlignmentPoolOutlivesStoreReaderAndExport) {
+TEST(PrimaryKeyRealtimeStoreTest, TestQueryPoolOutlivesStoreReaderAndExport) {
     const std::shared_ptr<arrow::Schema> stored_schema = PreparedSchema();
     std::shared_ptr<TestingMemoryPool> pool = std::make_shared<TestingMemoryPool>();
     std::weak_ptr<TestingMemoryPool> pool_lifetime = pool;
@@ -487,18 +361,13 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryAlignmentPoolOutlivesStoreReaderAndEx
         RealtimeWriteBatch{MakeBatch(R"([[0, 1, 0, 7, "seven"]])"), OffsetRange(0, 1)}));
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
 
-    arrow::FieldVector requested_fields = stored_schema->fields();
-    requested_fields.push_back(FieldWithId("added", arrow::int32(), 2));
     auto c_schema = std::make_unique<ArrowSchema>();
-    ASSERT_TRUE(
-        arrow::ExportSchema(*arrow::schema(std::move(requested_fields)), c_schema.get()).ok());
+    ASSERT_TRUE(arrow::ExportSchema(*stored_schema, c_schema.get()).ok());
     RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
                                  /*enable_predicate_pushdown=*/false};
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
                          store->CreateQueryReaders(view, /*offset_begin=*/0, context));
     ASSERT_EQ(1, readers.size());
-    ASSERT_GT(pool->allocation_count, 0);
-
     view.reset();
     store.reset();
     pool.reset();
@@ -516,7 +385,7 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryAlignmentPoolOutlivesStoreReaderAndEx
     ASSERT_TRUE(pool_lifetime.expired());
 }
 
-TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderAlignsNestedFieldsById) {
+TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderProjectsNestedFields) {
     const std::shared_ptr<arrow::Field> stored_profile_a =
         FieldWithId("profile_a", arrow::int32(), 30);
     const std::shared_ptr<arrow::Field> stored_a = FieldWithId("a", arrow::int32(), 10);
@@ -543,30 +412,13 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderAlignsNestedFieldsById) {
         OffsetRange(0, 1)}));
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
 
-    const std::shared_ptr<arrow::Field> requested_profile_missing =
-        FieldWithId("added_profile", arrow::int32(), 31);
-    const std::shared_ptr<arrow::Field> requested_profile_a =
-        FieldWithId("renamed_profile_a", arrow::int32(), 30);
-    const std::shared_ptr<arrow::Field> requested_b = FieldWithId("renamed_b", arrow::int32(), 11);
-    const std::shared_ptr<arrow::Field> requested_a = FieldWithId("renamed_a", arrow::int32(), 10);
-    const std::shared_ptr<arrow::Field> requested_item_missing =
-        FieldWithId("added_item", arrow::int32(), 12);
-    const std::shared_ptr<arrow::Field> requested_y = FieldWithId("renamed_y", arrow::int32(), 21);
-    const std::shared_ptr<arrow::Field> requested_x = FieldWithId("renamed_x", arrow::int32(), 20);
-    const std::shared_ptr<arrow::Field> requested_attr_missing =
-        FieldWithId("added_attr", arrow::int32(), 22);
     arrow::FieldVector requested_fields(stored_schema->fields().begin(),
                                         stored_schema->fields().begin() + 3);
-    requested_fields.push_back(FieldWithId(
-        "renamed_profile", arrow::struct_({requested_profile_missing, requested_profile_a}), 1));
-    requested_fields.push_back(FieldWithId(
-        "renamed_items",
-        arrow::list(arrow::struct_({requested_b, requested_item_missing, requested_a})), 2));
+    requested_fields.push_back(FieldWithId("profile", arrow::struct_({stored_profile_a}), 1));
+    requested_fields.push_back(
+        FieldWithId("items", arrow::list(arrow::struct_({stored_b, stored_a})), 2));
     requested_fields.push_back(
-        FieldWithId("renamed_attrs",
-                    arrow::map(arrow::utf8(),
-                               arrow::struct_({requested_y, requested_attr_missing, requested_x})),
-                    3));
+        FieldWithId("attrs", arrow::map(arrow::utf8(), arrow::struct_({stored_y, stored_x})), 3));
     std::shared_ptr<arrow::Schema> requested_schema = arrow::schema(std::move(requested_fields));
     auto c_schema = std::make_unique<ArrowSchema>();
     ASSERT_TRUE(arrow::ExportSchema(*requested_schema, c_schema.get()).ok());
@@ -583,15 +435,13 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderAlignsNestedFieldsById) {
     std::shared_ptr<arrow::StructArray> projected = checked_pointer_cast<arrow::StructArray>(array);
     const std::shared_ptr<arrow::StructArray> profile =
         checked_pointer_cast<arrow::StructArray>(projected->field(3));
-    ASSERT_TRUE(profile->field(0)->IsNull(0));
-    ASSERT_EQ(50, checked_pointer_cast<arrow::Int32Array>(profile->field(1))->Value(0));
+    ASSERT_EQ(50, checked_pointer_cast<arrow::Int32Array>(profile->field(0))->Value(0));
     const std::shared_ptr<arrow::ListArray> items =
         checked_pointer_cast<arrow::ListArray>(projected->field(4));
     const std::shared_ptr<arrow::StructArray> item_values =
         checked_pointer_cast<arrow::StructArray>(items->value_slice(0));
     ASSERT_EQ(200, checked_pointer_cast<arrow::Int32Array>(item_values->field(0))->Value(0));
-    ASSERT_TRUE(item_values->field(1)->IsNull(0));
-    ASSERT_EQ(100, checked_pointer_cast<arrow::Int32Array>(item_values->field(2))->Value(0));
+    ASSERT_EQ(100, checked_pointer_cast<arrow::Int32Array>(item_values->field(1))->Value(0));
     ASSERT_TRUE(item_values->IsNull(1));
 
     const std::shared_ptr<arrow::MapArray> attrs =
@@ -604,56 +454,9 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderAlignsNestedFieldsById) {
     const std::shared_ptr<arrow::StructArray> attr_values =
         checked_pointer_cast<arrow::StructArray>(attrs->items()->Slice(attr_offset, attr_length));
     ASSERT_EQ(8, checked_pointer_cast<arrow::Int32Array>(attr_values->field(0))->Value(0));
-    ASSERT_TRUE(attr_values->field(1)->IsNull(0));
-    ASSERT_EQ(7, checked_pointer_cast<arrow::Int32Array>(attr_values->field(2))->Value(0));
+    ASSERT_EQ(7, checked_pointer_cast<arrow::Int32Array>(attr_values->field(1))->Value(0));
     ASSERT_TRUE(attr_values->IsNull(1));
 }
 
-TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderRejectsMissingNonNullableNestedFields) {
-    const std::shared_ptr<arrow::Field> stored_profile_a =
-        FieldWithId("profile_a", arrow::int32(), 30);
-    const std::shared_ptr<arrow::Field> stored_item_a = FieldWithId("item_a", arrow::int32(), 10);
-    const std::shared_ptr<arrow::Field> stored_attr_a = FieldWithId("attr_a", arrow::int32(), 20);
-    arrow::FieldVector stored_fields = {
-        DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
-        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
-            ->WithNullable(false),
-        DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()),
-        FieldWithId("profile", arrow::struct_({stored_profile_a}), 1),
-        FieldWithId("items", arrow::list(arrow::struct_({stored_item_a})), 2),
-        FieldWithId("attrs", arrow::map(arrow::utf8(), arrow::struct_({stored_attr_a})), 3)};
-    const std::shared_ptr<arrow::Schema> stored_schema = arrow::schema(std::move(stored_fields));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(stored_schema, GetDefaultPool()));
-    ASSERT_OK(store->Write(
-        RealtimeWriteBatch{MakeBatch(stored_schema, R"([[0, 1, 0, [5], [[10]], [["key", [20]]]]])"),
-                           OffsetRange(0, 1)}));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
-
-    const std::shared_ptr<arrow::Field> required =
-        FieldWithId("required_nested", arrow::int32(), 99, /*nullable=*/false);
-    std::vector<arrow::FieldVector> requested_schemas;
-    arrow::FieldVector struct_fields = stored_schema->fields();
-    struct_fields[3] = FieldWithId("profile", arrow::struct_({stored_profile_a, required}), 1);
-    requested_schemas.push_back(std::move(struct_fields));
-    arrow::FieldVector list_fields = stored_schema->fields();
-    list_fields[4] =
-        FieldWithId("items", arrow::list(arrow::struct_({stored_item_a, required})), 2);
-    requested_schemas.push_back(std::move(list_fields));
-    arrow::FieldVector map_fields = stored_schema->fields();
-    map_fields[5] = FieldWithId(
-        "attrs", arrow::map(arrow::utf8(), arrow::struct_({stored_attr_a, required})), 3);
-    requested_schemas.push_back(std::move(map_fields));
-
-    for (const arrow::FieldVector& fields : requested_schemas) {
-        auto c_schema = std::make_unique<ArrowSchema>();
-        ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(fields), c_schema.get()).ok());
-        RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
-                                     /*enable_predicate_pushdown=*/false};
-        ASSERT_NOK_WITH_MSG(store->CreateQueryReaders(view, /*offset_begin=*/0, context),
-                            "requested non-nullable field 'required_nested' with id 99 is absent");
-    }
-}
-
 }  // namespace
 }  // namespace paimon::test
diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp
index 85a9e83b3..66a3f7426 100644
--- a/src/paimon/core/table/source/table_scan.cpp
+++ b/src/paimon/core/table/source/table_scan.cpp
@@ -41,7 +41,6 @@
 #include "paimon/core/operation/data_evolution_file_store_scan.h"
 #include "paimon/core/operation/file_store_scan.h"
 #include "paimon/core/operation/key_value_file_store_scan.h"
-#include "paimon/core/realtime/framework/primary_key_realtime_validator.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
 #include "paimon/core/schema/schema_manager.h"
 #include "paimon/core/schema/schema_validation.h"
@@ -64,6 +63,7 @@
 #include "paimon/core/utils/field_mapping.h"
 #include "paimon/core/utils/file_store_path_factory.h"
 #include "paimon/core/utils/index_file_path_factories.h"
+#include "paimon/core/utils/primary_key_table_utils.h"
 #include "paimon/core/utils/snapshot_manager.h"
 #include "paimon/format/file_format.h"
 #include "paimon/realtime/realtime_context.h"
@@ -238,7 +238,7 @@ Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions&
                 "PK real-time union read does not support read-optimized scans");
         }
         PAIMON_RETURN_NOT_OK(
-            PrimaryKeyRealtimeValidator::ValidateOptions(core_options, table_schema));
+            PrimaryKeyTableUtils::ValidateRealtimeOptions(core_options, table_schema));
     }
     if (context.IsStreamingMode()) {
         return Status::Invalid("real-time union read currently supports batch scans only");
diff --git a/src/paimon/core/utils/primary_key_table_utils.cpp b/src/paimon/core/utils/primary_key_table_utils.cpp
index cf72da4ae..446cdc4b3 100644
--- a/src/paimon/core/utils/primary_key_table_utils.cpp
+++ b/src/paimon/core/utils/primary_key_table_utils.cpp
@@ -29,12 +29,14 @@
 #include "paimon/common/utils/fields_comparator.h"
 #include "paimon/common/utils/object_utils.h"
 #include "paimon/core/core_options.h"
+#include "paimon/core/index/pk/primary_key_index_definitions.h"
 #include "paimon/core/mergetree/compact/aggregate/aggregate_merge_function.h"
 #include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
 #include "paimon/core/mergetree/compact/first_row_merge_function.h"
 #include "paimon/core/mergetree/compact/merge_function.h"
 #include "paimon/core/mergetree/compact/partial_update_merge_function.h"
 #include "paimon/core/options/merge_engine.h"
+#include "paimon/core/schema/table_schema.h"
 #include "paimon/status.h"
 
 namespace arrow {
@@ -96,4 +98,52 @@ Result<std::unique_ptr<FieldsComparator>> PrimaryKeyTableUtils::CreateSequenceFi
                                     options.SequenceFieldSortOrderIsAscending());
 }
 
+Status PrimaryKeyTableUtils::ValidateRealtimeOptions(const CoreOptions& options,
+                                                     const TableSchema& schema) {
+    if (options.GetBucket() <= 0) {
+        return Status::NotImplemented("PK realtime v1 requires fixed buckets");
+    }
+    if (options.GetMergeEngine() != MergeEngine::DEDUPLICATE) {
+        return Status::NotImplemented("PK realtime v1 supports only the DEDUPLICATE merge engine");
+    }
+    if (options.DataEvolutionEnabled()) {
+        return Status::NotImplemented("PK realtime v1 does not support data evolution");
+    }
+    if (!options.GetFieldsSequenceGroups().empty()) {
+        return Status::NotImplemented("PK realtime v1 does not support sequence groups");
+    }
+    if (options.IgnoreDelete() || options.PartialUpdateRemoveRecordOnDelete() ||
+        options.AggregationRemoveRecordOnDelete() ||
+        !options.GetPartialUpdateRemoveRecordOnSequenceGroup().empty()) {
+        return Status::NotImplemented("PK realtime v1 requires default delete behavior");
+    }
+    if (!options.GetSequenceField().empty()) {
+        return Status::NotImplemented("PK realtime v1 does not support sequence.field");
+    }
+    if (!options.SequenceFieldSortOrderIsAscending()) {
+        return Status::NotImplemented(
+            "PK realtime v1 supports only ascending sequence.field.sort-order");
+    }
+    if (options.NeedLookup() || options.DeletionVectorsEnabled() ||
+        options.GetChangelogProducer() != ChangelogProducer::NONE) {
+        return Status::NotImplemented("PK realtime v1 does not support lookup or early MOR");
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::vector<DataField> primary_key_fields,
+                           schema.TrimmedPrimaryKeyFields());
+    for (const DataField& field : primary_key_fields) {
+        if (field.Type()->id() == arrow::Type::FLOAT || field.Type()->id() == arrow::Type::DOUBLE) {
+            return Status::NotImplemented(
+                "PK realtime v1 does not support FLOAT or DOUBLE primary keys");
+        }
+    }
+    if (options.GlobalIndexEnabled()) {
+        PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions definitions,
+                               PrimaryKeyIndexDefinitions::Create(schema));
+        if (!definitions.Definitions().empty()) {
+            return Status::NotImplemented("PK realtime v1 does not support global indexes");
+        }
+    }
+    return Status::OK();
+}
+
 }  // namespace paimon
diff --git a/src/paimon/core/utils/primary_key_table_utils.h b/src/paimon/core/utils/primary_key_table_utils.h
index 82a108ab7..114801cc1 100644
--- a/src/paimon/core/utils/primary_key_table_utils.h
+++ b/src/paimon/core/utils/primary_key_table_utils.h
@@ -36,6 +36,7 @@ class CoreOptions;
 class MemoryPool;
 class FieldsComparator;
 class DataField;
+class TableSchema;
 
 class PrimaryKeyTableUtils {
  public:
@@ -57,6 +58,8 @@ class PrimaryKeyTableUtils {
 
     static Result<std::unique_ptr<FieldsComparator>> CreateSequenceFieldsComparator(
         const std::vector<DataField>& value_fields, const CoreOptions& options);
+
+    static Status ValidateRealtimeOptions(const CoreOptions& options, const TableSchema& schema);
 };
 
 }  // namespace paimon
diff --git a/src/paimon/core/utils/primary_key_table_utils_test.cpp b/src/paimon/core/utils/primary_key_table_utils_test.cpp
index 1a7345fdf..5f35f78a2 100644
--- a/src/paimon/core/utils/primary_key_table_utils_test.cpp
+++ b/src/paimon/core/utils/primary_key_table_utils_test.cpp
@@ -19,6 +19,7 @@
 #include "paimon/core/utils/primary_key_table_utils.h"
 
 #include <cstdint>
+#include <map>
 #include <memory>
 #include <optional>
 #include <string>
@@ -33,6 +34,7 @@
 #include "paimon/core/core_options.h"
 #include "paimon/core/key_value.h"
 #include "paimon/core/mergetree/compact/merge_function.h"
+#include "paimon/core/schema/table_schema.h"
 #include "paimon/defs.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/status.h"
@@ -41,6 +43,62 @@
 #include "paimon/testing/utils/testharness.h"
 
 namespace paimon::test {
+namespace {
+
+std::shared_ptr<TableSchema> PkSchema(
+    const std::shared_ptr<arrow::DataType>& key_type = arrow::int64(),
+    const std::map<std::string, std::string>& options = {}) {
+    return TableSchema::Create(
+               /*schema_id=*/0,
+               arrow::schema({arrow::field("id", key_type), arrow::field("value", arrow::utf8())}),
+               /*partition_keys=*/{}, /*primary_keys=*/{"id"}, options)
+        .value();
+}
+
+}  // namespace
+
+TEST(PrimaryKeyTableUtilsTest, TestSupportedRealtimeOptions) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
+    ASSERT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *PkSchema()));
+}
+
+TEST(PrimaryKeyTableUtilsTest, TestUnsupportedRealtimeOptions) {
+    const std::string sequence_group =
+        std::string(Options::FIELDS_PREFIX) + ".value." + Options::SEQUENCE_GROUP;
+    const std::vector<std::map<std::string, std::string>> unsupported_options = {
+        {{Options::BUCKET, "0"}},
+        {{Options::BUCKET, "1"}, {Options::MERGE_ENGINE, "partial-update"}},
+        {{Options::BUCKET, "1"}, {Options::DATA_EVOLUTION_ENABLED, "true"}},
+        {{Options::BUCKET, "1"}, {sequence_group, "seq"}},
+        {{Options::BUCKET, "1"}, {Options::SEQUENCE_FIELD, "seq"}},
+        {{Options::BUCKET, "1"}, {Options::FORCE_LOOKUP, "true"}},
+        {{Options::BUCKET, "1"}, {Options::DELETION_VECTORS_ENABLED, "true"}},
+        {{Options::BUCKET, "1"}, {Options::CHANGELOG_PRODUCER, "input"}},
+    };
+    for (const std::map<std::string, std::string>& option_map : unsupported_options) {
+        ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
+        ASSERT_NOK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *PkSchema()));
+    }
+}
+
+TEST(PrimaryKeyTableUtilsTest, TestRealtimeRejectsFloatingPrimaryKeys) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
+    ASSERT_NOK_WITH_MSG(
+        PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *PkSchema(arrow::float32())),
+        "FLOAT or DOUBLE primary keys");
+    ASSERT_NOK_WITH_MSG(
+        PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *PkSchema(arrow::float64())),
+        "FLOAT or DOUBLE primary keys");
+}
+
+TEST(PrimaryKeyTableUtilsTest, TestRealtimeRejectsEnabledGlobalIndex) {
+    const std::map<std::string, std::string> option_map = {{Options::BUCKET, "1"},
+                                                           {Options::PK_BTREE_INDEX_COLUMNS, "id"}};
+    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
+    ASSERT_NOK_WITH_MSG(PrimaryKeyTableUtils::ValidateRealtimeOptions(
+                            options, *PkSchema(arrow::int64(), option_map)),
+                        "does not support global indexes");
+}
 
 TEST(PrimaryKeyTableUtilsTest, TestCreateSequenceFieldsComparator) {
     {
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index 94bf81edc..62db90296 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -1882,45 +1882,6 @@ TEST_F(RealtimeWriteInteTest, TestPkNestedProjectionAcrossDiskAndMemory) {
     ASSERT_OK(writer->Close());
 }
 
-TEST_F(RealtimeWriteInteTest, TestPkMemoryReadAfterSchemaEvolution) {
-    CreatePkTable();
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                         CreateRealtimeWriter(realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
-                         MakeBatch({Row{1, "old", "p0"}}, /*partitioned=*/false));
-    ASSERT_OK(writer->Write(std::move(batch)));
-
-    std::shared_ptr<arrow::Field> renamed_payload = arrow::field("renamed_payload", arrow::utf8());
-    std::shared_ptr<arrow::Field> added = arrow::field("added", arrow::int32());
-    ASSERT_OK(TestHelper::WriteNextSchema(dir_->GetFileSystem(), table_path_,
-                                          {DataField(0, fields_[0]), DataField(1, renamed_payload),
-                                           DataField(2, fields_[2]), DataField(3, added)},
-                                          /*highest_field_id=*/3, options_));
-    fields_[1] = renamed_payload;
-    fields_.push_back(added);
-    schema_ = arrow::schema(fields_);
-
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
-                         CreatePlan(realtime_context, /*predicate=*/nullptr));
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult result,
-                         ReadPlan(plan, realtime_context, {"id", "renamed_payload", "pt", "added"},
-                                  /*predicate=*/nullptr, /*enable_predicate_filter=*/false));
-    ASSERT_EQ(1, result.data->num_chunks());
-    std::shared_ptr<arrow::StructArray> row =
-        std::dynamic_pointer_cast<arrow::StructArray>(result.data->chunk(0));
-    ASSERT_NE(nullptr, row);
-    ASSERT_EQ(1, row->length());
-    std::shared_ptr<arrow::StringArray> renamed_values =
-        std::dynamic_pointer_cast<arrow::StringArray>(row->field(2));
-    ASSERT_NE(nullptr, renamed_values);
-    ASSERT_EQ("old", renamed_values->GetString(0));
-    ASSERT_TRUE(row->field(4)->IsNull(0));
-    result.reader->Close();
-    ASSERT_OK(writer->Close());
-}
-
 TEST_F(RealtimeWriteInteTest, TestPkCompositeMerge) {
     CreatePkTable(/*partition_keys=*/{}, /*primary_keys=*/{"id", "payload"});
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,

From 32844c103a5e71cc542e42186db933f78dc81376 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Wed, 26 Aug 2026 14:15:22 +0800
Subject: [PATCH 40/62] refactor(realtime): centralize Arrow array memory
 retention

---
 src/paimon/CMakeLists.txt                     |  1 -
 src/paimon/common/utils/arrow/mem_utils.cpp   | 41 +++++++++++
 src/paimon/common/utils/arrow/mem_utils.h     |  6 ++
 .../core/realtime/arrow_array_pool_holder.cpp | 69 -------------------
 .../core/realtime/arrow_array_pool_holder.h   | 36 ----------
 .../realtime/primary_key_realtime_store.cpp   |  1 -
 .../primary_key_realtime_store_test.cpp       | 13 +++-
 .../realtime/realtime_primary_key_writer.cpp  |  1 -
 8 files changed, 59 insertions(+), 109 deletions(-)
 delete mode 100644 src/paimon/core/realtime/arrow_array_pool_holder.cpp
 delete mode 100644 src/paimon/core/realtime/arrow_array_pool_holder.h

diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index 55d033570..0a78b0902 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -380,7 +380,6 @@ set(PAIMON_CORE_SRCS
     core/manifest/snapshot_live_manifest_entries.cpp
     core/operation/write_context.cpp
     core/operation/write_restore.cpp
-    core/realtime/arrow_array_pool_holder.cpp
     core/realtime/arrow_realtime_store.cpp
     core/realtime/arrow_realtime_store_factory.cpp
     core/realtime/prepared_key_value_reader.cpp
diff --git a/src/paimon/common/utils/arrow/mem_utils.cpp b/src/paimon/common/utils/arrow/mem_utils.cpp
index 7e8986be5..1e9695332 100644
--- a/src/paimon/common/utils/arrow/mem_utils.cpp
+++ b/src/paimon/common/utils/arrow/mem_utils.cpp
@@ -24,12 +24,31 @@
 #include <new>
 #include <string>
 
+#include "arrow/c/abi.h"
+#include "arrow/c/helpers.h"
 #include "arrow/memory_pool.h"
 #include "arrow/status.h"
 #include "fmt/format.h"
 #include "paimon/memory/memory_pool.h"
 
 namespace paimon {
+namespace {
+
+struct ArrowArrayPrivateData {
+    void (*release)(ArrowArray*);
+    void* private_data;
+    std::shared_ptr<arrow::MemoryPool> arrow_pool;
+};
+
+void ReleaseArrowArray(ArrowArray* array) {
+    std::unique_ptr<ArrowArrayPrivateData> data(
+        static_cast<ArrowArrayPrivateData*>(array->private_data));
+    array->release = data->release;
+    array->private_data = data->private_data;
+    array->release(array);
+}
+
+}  // namespace
 
 class ArrowMemPoolAdaptor : public arrow::MemoryPool {
  public:
@@ -107,4 +126,26 @@ std::unique_ptr<arrow::MemoryPool> GetArrowPool(const std::shared_ptr<MemoryPool
     return std::make_unique<ArrowMemPoolAdaptor>(pool);
 }
 
+Status RetainArrowArrayMemoryPool(ArrowArray* array,
+                                  const std::shared_ptr<arrow::MemoryPool>& arrow_pool) {
+    if (!array || !array->release) {
+        return Status::Invalid("cannot retain Arrow array memory pool");
+    }
+    if (!arrow_pool) {
+        ArrowArrayRelease(array);
+        return Status::Invalid("cannot retain Arrow array memory pool");
+    }
+    std::unique_ptr<ArrowArrayPrivateData> data;
+    try {
+        data = std::make_unique<ArrowArrayPrivateData>(
+            ArrowArrayPrivateData{array->release, array->private_data, arrow_pool});
+    } catch (const std::bad_alloc&) {
+        ArrowArrayRelease(array);
+        return Status::OutOfMemory("failed to retain Arrow array memory pool");
+    }
+    array->private_data = data.release();
+    array->release = ReleaseArrowArray;
+    return Status::OK();
+}
+
 }  // namespace paimon
diff --git a/src/paimon/common/utils/arrow/mem_utils.h b/src/paimon/common/utils/arrow/mem_utils.h
index 96b59e3e8..214bb4509 100644
--- a/src/paimon/common/utils/arrow/mem_utils.h
+++ b/src/paimon/common/utils/arrow/mem_utils.h
@@ -23,11 +23,17 @@
 
 #include "arrow/memory_pool.h"
 #include "paimon/memory/memory_pool.h"
+#include "paimon/status.h"
 #include "paimon/visibility.h"
 
+struct ArrowArray;
+
 namespace paimon {
 
 PAIMON_EXPORT std::unique_ptr<arrow::MemoryPool> GetArrowPool(
     const std::shared_ptr<MemoryPool>& pool);
 
+Status RetainArrowArrayMemoryPool(ArrowArray* array,
+                                  const std::shared_ptr<arrow::MemoryPool>& arrow_pool);
+
 }  // namespace paimon
diff --git a/src/paimon/core/realtime/arrow_array_pool_holder.cpp b/src/paimon/core/realtime/arrow_array_pool_holder.cpp
deleted file mode 100644
index 97a01dd19..000000000
--- a/src/paimon/core/realtime/arrow_array_pool_holder.cpp
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "paimon/core/realtime/arrow_array_pool_holder.h"
-
-#include <memory>
-#include <new>
-
-#include "arrow/c/abi.h"
-#include "arrow/c/helpers.h"
-#include "arrow/memory_pool.h"
-
-namespace paimon {
-namespace {
-
-struct ArrowArrayPrivateData {
-    void (*release)(ArrowArray*);
-    void* private_data;
-    std::shared_ptr<arrow::MemoryPool> arrow_pool;
-};
-
-void ReleaseArrowArray(ArrowArray* array) {
-    std::unique_ptr<ArrowArrayPrivateData> data(
-        static_cast<ArrowArrayPrivateData*>(array->private_data));
-    array->release = data->release;
-    array->private_data = data->private_data;
-    array->release(array);
-}
-
-}  // namespace
-
-Status RetainArrowArrayMemoryPool(ArrowArray* array,
-                                  const std::shared_ptr<arrow::MemoryPool>& arrow_pool) {
-    if (!array || !array->release) {
-        return Status::Invalid("cannot retain Arrow array memory pool");
-    }
-    if (!arrow_pool) {
-        ArrowArrayRelease(array);
-        return Status::Invalid("cannot retain Arrow array memory pool");
-    }
-    std::unique_ptr<ArrowArrayPrivateData> data;
-    try {
-        data = std::make_unique<ArrowArrayPrivateData>(
-            ArrowArrayPrivateData{array->release, array->private_data, arrow_pool});
-    } catch (const std::bad_alloc&) {
-        ArrowArrayRelease(array);
-        return Status::OutOfMemory("failed to retain Arrow array memory pool");
-    }
-    array->private_data = data.release();
-    array->release = ReleaseArrowArray;
-    return Status::OK();
-}
-
-}  // namespace paimon
diff --git a/src/paimon/core/realtime/arrow_array_pool_holder.h b/src/paimon/core/realtime/arrow_array_pool_holder.h
deleted file mode 100644
index 05f9ed9dc..000000000
--- a/src/paimon/core/realtime/arrow_array_pool_holder.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-
-#include <memory>
-
-#include "paimon/status.h"
-
-struct ArrowArray;
-
-namespace arrow {
-class MemoryPool;
-}  // namespace arrow
-
-namespace paimon {
-
-Status RetainArrowArrayMemoryPool(ArrowArray* array,
-                                  const std::shared_ptr<arrow::MemoryPool>& arrow_pool);
-
-}  // namespace paimon
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index 4e1defdf5..847347113 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -32,7 +32,6 @@
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/scope_guard.h"
-#include "paimon/core/realtime/arrow_array_pool_holder.h"
 #include "paimon/core/realtime/prepared_key_value_reader.h"
 #include "paimon/core/utils/nested_projection_utils.h"
 #include "paimon/macros.h"
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index 5f2e46928..209d2ae7a 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -318,9 +318,20 @@ TEST(PrimaryKeyRealtimeStoreTest, TestReclaimKeepsReadView) {
         store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 0, 4, 1, "one"]])"), OffsetRange(4, 5)}));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
                          store->SealForCommit());
+    ASSERT_TRUE(segment.has_value());
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
+    ASSERT_GT(store->GetMemoryUsage(), 0);
     ASSERT_OK(store->AdvanceCommittedOffset(5));
-    ASSERT_EQ(std::optional<OffsetRange>(OffsetRange(4, 5)), view->GetOffsetRange());
+    ASSERT_EQ(0, store->GetMemoryUsage());
+
+    auto c_schema = std::make_unique<ArrowSchema>();
+    ASSERT_TRUE(arrow::ExportSchema(*PreparedSchema(), c_schema.get()).ok());
+    RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store->CreateQueryReaders(view, /*offset_begin=*/0, context));
+    ASSERT_OK_AND_ASSIGN(std::string actual, ReadJson(readers));
+    ASSERT_NE(std::string::npos, actual.find("\"one\""));
 }
 
 TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderPerStoredBatch) {
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index d2b2b86ce..89e86e787 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -36,7 +36,6 @@
 #include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
 #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
 #include "paimon/core/mergetree/merge_tree_writer.h"
-#include "paimon/core/realtime/arrow_array_pool_holder.h"
 #include "paimon/core/realtime/prepared_key_value_reader.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
 #include "paimon/core/utils/commit_increment.h"

From 3476342bca2e3d8f93ed4cad78a23933b59c1202 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Wed, 26 Aug 2026 15:45:55 +0800
Subject: [PATCH 41/62] fix(realtime): refine primary key option validation

---
 .../core/utils/primary_key_table_utils.cpp    | 18 +++---
 .../utils/primary_key_table_utils_test.cpp    | 60 +++++++++++++++++--
 2 files changed, 63 insertions(+), 15 deletions(-)

diff --git a/src/paimon/core/utils/primary_key_table_utils.cpp b/src/paimon/core/utils/primary_key_table_utils.cpp
index 446cdc4b3..b5efb5ded 100644
--- a/src/paimon/core/utils/primary_key_table_utils.cpp
+++ b/src/paimon/core/utils/primary_key_table_utils.cpp
@@ -109,12 +109,7 @@ Status PrimaryKeyTableUtils::ValidateRealtimeOptions(const CoreOptions& options,
     if (options.DataEvolutionEnabled()) {
         return Status::NotImplemented("PK realtime v1 does not support data evolution");
     }
-    if (!options.GetFieldsSequenceGroups().empty()) {
-        return Status::NotImplemented("PK realtime v1 does not support sequence groups");
-    }
-    if (options.IgnoreDelete() || options.PartialUpdateRemoveRecordOnDelete() ||
-        options.AggregationRemoveRecordOnDelete() ||
-        !options.GetPartialUpdateRemoveRecordOnSequenceGroup().empty()) {
+    if (options.IgnoreDelete()) {
         return Status::NotImplemented("PK realtime v1 requires default delete behavior");
     }
     if (!options.GetSequenceField().empty()) {
@@ -124,9 +119,14 @@ Status PrimaryKeyTableUtils::ValidateRealtimeOptions(const CoreOptions& options,
         return Status::NotImplemented(
             "PK realtime v1 supports only ascending sequence.field.sort-order");
     }
-    if (options.NeedLookup() || options.DeletionVectorsEnabled() ||
-        options.GetChangelogProducer() != ChangelogProducer::NONE) {
-        return Status::NotImplemented("PK realtime v1 does not support lookup or early MOR");
+    if (options.GetChangelogProducer() != ChangelogProducer::NONE) {
+        return Status::NotImplemented("PK realtime v1 supports only the NONE changelog producer");
+    }
+    if (options.DeletionVectorsEnabled()) {
+        return Status::NotImplemented("PK realtime v1 does not support deletion vectors");
+    }
+    if (options.NeedLookup()) {
+        return Status::NotImplemented("PK realtime v1 does not support lookup");
     }
     PAIMON_ASSIGN_OR_RAISE(std::vector<DataField> primary_key_fields,
                            schema.TrimmedPrimaryKeyFields());
diff --git a/src/paimon/core/utils/primary_key_table_utils_test.cpp b/src/paimon/core/utils/primary_key_table_utils_test.cpp
index 5f35f78a2..8887102cd 100644
--- a/src/paimon/core/utils/primary_key_table_utils_test.cpp
+++ b/src/paimon/core/utils/primary_key_table_utils_test.cpp
@@ -63,17 +63,11 @@ TEST(PrimaryKeyTableUtilsTest, TestSupportedRealtimeOptions) {
 }
 
 TEST(PrimaryKeyTableUtilsTest, TestUnsupportedRealtimeOptions) {
-    const std::string sequence_group =
-        std::string(Options::FIELDS_PREFIX) + ".value." + Options::SEQUENCE_GROUP;
     const std::vector<std::map<std::string, std::string>> unsupported_options = {
         {{Options::BUCKET, "0"}},
         {{Options::BUCKET, "1"}, {Options::MERGE_ENGINE, "partial-update"}},
         {{Options::BUCKET, "1"}, {Options::DATA_EVOLUTION_ENABLED, "true"}},
-        {{Options::BUCKET, "1"}, {sequence_group, "seq"}},
         {{Options::BUCKET, "1"}, {Options::SEQUENCE_FIELD, "seq"}},
-        {{Options::BUCKET, "1"}, {Options::FORCE_LOOKUP, "true"}},
-        {{Options::BUCKET, "1"}, {Options::DELETION_VECTORS_ENABLED, "true"}},
-        {{Options::BUCKET, "1"}, {Options::CHANGELOG_PRODUCER, "input"}},
     };
     for (const std::map<std::string, std::string>& option_map : unsupported_options) {
         ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
@@ -81,6 +75,60 @@ TEST(PrimaryKeyTableUtilsTest, TestUnsupportedRealtimeOptions) {
     }
 }
 
+TEST(PrimaryKeyTableUtilsTest, TestRealtimeAcceptsInactiveMergeEngineOptions) {
+    const std::string sequence_group =
+        std::string(Options::FIELDS_PREFIX) + ".value." + Options::SEQUENCE_GROUP;
+    const std::map<std::string, std::string> option_map = {
+        {Options::BUCKET, "1"},
+        {sequence_group, "seq"},
+        {Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, "true"},
+        {Options::AGGREGATION_REMOVE_RECORD_ON_DELETE, "true"},
+        {Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_SEQUENCE_GROUP, "seq"},
+    };
+    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<TableSchema> schema,
+                         TableSchema::Create(0,
+                                             arrow::schema({arrow::field("id", arrow::int64()),
+                                                            arrow::field("value", arrow::utf8()),
+                                                            arrow::field("seq", arrow::int64())}),
+                                             {}, {"id"}, option_map));
+    ASSERT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *schema));
+}
+
+TEST(PrimaryKeyTableUtilsTest, TestRealtimeRejectsDeleteAndSequenceOrderingOptions) {
+    ASSERT_OK_AND_ASSIGN(
+        CoreOptions ignore_delete,
+        CoreOptions::FromMap({{Options::BUCKET, "1"}, {Options::IGNORE_DELETE, "true"}}));
+    ASSERT_NOK_WITH_MSG(PrimaryKeyTableUtils::ValidateRealtimeOptions(ignore_delete, *PkSchema()),
+                        "requires default delete behavior");
+
+    ASSERT_OK_AND_ASSIGN(
+        CoreOptions descending,
+        CoreOptions::FromMap(
+            {{Options::BUCKET, "1"}, {Options::SEQUENCE_FIELD_SORT_ORDER, "descending"}}));
+    ASSERT_NOK_WITH_MSG(PrimaryKeyTableUtils::ValidateRealtimeOptions(descending, *PkSchema()),
+                        "supports only ascending sequence.field.sort-order");
+}
+
+TEST(PrimaryKeyTableUtilsTest, TestRealtimeReportsSpecificLookupErrors) {
+    const std::vector<std::pair<std::map<std::string, std::string>, std::string>> cases = {
+        {{{Options::BUCKET, "1"}, {Options::FORCE_LOOKUP, "true"}},
+         "PK realtime v1 does not support lookup"},
+        {{{Options::BUCKET, "1"}, {Options::DELETION_VECTORS_ENABLED, "true"}},
+         "PK realtime v1 does not support deletion vectors"},
+        {{{Options::BUCKET, "1"}, {Options::CHANGELOG_PRODUCER, "input"}},
+         "PK realtime v1 supports only the NONE changelog producer"},
+        {{{Options::BUCKET, "1"}, {Options::CHANGELOG_PRODUCER, "lookup"}},
+         "PK realtime v1 supports only the NONE changelog producer"},
+    };
+    for (const auto& [option_map, expected_message] : cases) {
+        ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
+        Status status = PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *PkSchema());
+        ASSERT_TRUE(status.IsNotImplemented());
+        ASSERT_EQ(status.message(), expected_message);
+    }
+}
+
 TEST(PrimaryKeyTableUtilsTest, TestRealtimeRejectsFloatingPrimaryKeys) {
     ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
     ASSERT_NOK_WITH_MSG(

From 961aaaacbe814d00c1cedd66f9b1be77c14a8bfe Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Wed, 26 Aug 2026 16:49:01 +0800
Subject: [PATCH 42/62] fix(realtime): validate plugin bitmap bounds

---
 .../merged_key_value_record_reader_test.cpp   | 52 +++++++++++++++++++
 .../realtime/prepared_key_value_reader.cpp    | 14 ++++-
 2 files changed, 64 insertions(+), 2 deletions(-)

diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index bf1aaec9e..1a94aee0e 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -98,6 +98,36 @@ class TrackingBatchReader : public BatchReader {
     int32_t* close_count_;
 };
 
+class MalformedBitmapBatchReader : public BatchReader {
+ public:
+    MalformedBitmapBatchReader(std::unique_ptr<BatchReader>&& delegate, int32_t row_id)
+        : delegate_(std::move(delegate)), row_id_(row_id) {}
+
+    Result<ReadBatch> NextBatch() override {
+        return delegate_->NextBatch();
+    }
+
+    Result<ReadBatchWithBitmap> NextBatchWithBitmap() override {
+        PAIMON_ASSIGN_OR_RAISE(ReadBatchWithBitmap batch, delegate_->NextBatchWithBitmap());
+        if (!IsEofBatch(batch)) {
+            batch.second.Add(row_id_);
+        }
+        return batch;
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return delegate_->GetReaderMetrics();
+    }
+
+    void Close() override {
+        delegate_->Close();
+    }
+
+ private:
+    std::unique_ptr<BatchReader> delegate_;
+    int32_t row_id_;
+};
+
 }  // namespace
 
 class MergedKeyValueRecordReaderTest : public testing::Test {
@@ -259,6 +289,28 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderRejectsReversedVisibleO
     ASSERT_NOK_WITH_MSG(result, "prepared visible offset range begin exceeds end");
 }
 
+TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderBitmapBounds) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1]])").ValueOrDie();
+    auto batch_reader = std::make_unique<MalformedBitmapBatchReader>(
+        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, /*batch_size=*/1),
+        /*row_id=*/1);
+
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
+                                        value_schema, value_schema, pool_));
+    Result<std::vector<KeyValue>> result =
+        ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
+                                                   KeyValueRecordReader::Iterator>(reader.get());
+    ASSERT_TRUE(result.status().IsInvalid());
+    ASSERT_NOK_WITH_MSG(result, "selected row id 1 is out of bounds for prepared batch length 1");
+}
+
 TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderCommitSchema) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Field> extra = MakeField("extra", arrow::int32(), 1);
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index ed1f9e158..2a2251784 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -321,7 +321,9 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
             }
             key_ctx_ = std::make_shared<ColumnarBatchContext>(key_fields, pool_);
             value_ctx_ = std::make_shared<ColumnarBatchContext>(value_fields, pool_);
-            if (!SelectRows(*offset_array, std::move(selection))) {
+            PAIMON_ASSIGN_OR_RAISE(bool has_selected_rows,
+                                   SelectRows(*offset_array, std::move(selection)));
+            if (!has_selected_rows) {
                 continue;
             }
             ArrowUtils::TraverseArray(data_batch);
@@ -362,7 +364,15 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
         return Status::OK();
     }
 
-    bool SelectRows(const arrow::Int64Array& offsets, RoaringBitmap32&& selection) {
+    Result<bool> SelectRows(const arrow::Int64Array& offsets, RoaringBitmap32&& selection) {
+        for (auto iter = selection.Begin(); iter != selection.End(); ++iter) {
+            const int32_t row = *iter;
+            if (row < 0 || row >= offsets.length()) {
+                return Status::Invalid(
+                    fmt::format("selected row id {} is out of bounds for prepared batch length {}",
+                                row, offsets.length()));
+            }
+        }
         if (!visible_offsets_.has_value()) {
             selected_rows_.reserve(offsets.length());
             for (int64_t row = 0; row < offsets.length(); ++row) {

From 69d9693f6d1f72ddf059378db8a69e3f4793ddb5 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Wed, 26 Aug 2026 18:41:09 +0800
Subject: [PATCH 43/62] fix(realtime): preserve primary keys in projected reads

---
 src/paimon/common/table/special_fields.h      | 15 +++++
 .../common/table/special_fields_test.cpp      | 21 ++++++
 .../realtime/prepared_key_value_reader.cpp    | 65 ++++++++++---------
 .../realtime/realtime_primary_key_writer.cpp  | 15 ++---
 .../table/source/key_value_table_read.cpp     | 28 +++++---
 test/inte/realtime_write_inte_test.cpp        | 54 +++++++++++++++
 6 files changed, 148 insertions(+), 50 deletions(-)

diff --git a/src/paimon/common/table/special_fields.h b/src/paimon/common/table/special_fields.h
index 9771ac232..0e07882a8 100644
--- a/src/paimon/common/table/special_fields.h
+++ b/src/paimon/common/table/special_fields.h
@@ -21,6 +21,7 @@
 #include <cstdint>
 #include <limits>
 #include <string>
+#include <utility>
 
 #include "arrow/type_fwd.h"
 #include "paimon/common/types/data_field.h"
@@ -35,6 +36,10 @@ struct SpecialFields {
 
     static constexpr char KEY_FIELD_PREFIX[] = "_KEY_";
     static constexpr int32_t KEY_VALUE_SPECIAL_FIELD_COUNT = 2;
+    static constexpr int32_t kPreparedKeyValueValueKindIndex = 0;
+    static constexpr int32_t kPreparedKeyValueSequenceNumberIndex = 1;
+    static constexpr int32_t kPreparedKeyValueRealtimeOffsetIndex = 2;
+    static constexpr int32_t kPreparedKeyValueValueStartIndex = 3;
 
     static const DataField& SequenceNumber() {
         static const DataField data_field = DataField(
@@ -92,6 +97,16 @@ struct SpecialFields {
         target_fields.insert(target_fields.end(), schema->fields().begin(), schema->fields().end());
         return arrow::schema(target_fields);
     }
+
+    static std::shared_ptr<arrow::Schema> PreparedKeyValueSchema(
+        const arrow::FieldVector& value_fields) {
+        arrow::FieldVector fields = {
+            DataField::ConvertDataFieldToArrowField(ValueKind())->WithNullable(false),
+            DataField::ConvertDataFieldToArrowField(SequenceNumber())->WithNullable(false),
+            DataField::ConvertDataFieldToArrowField(RealtimeOffset())};
+        fields.insert(fields.end(), value_fields.begin(), value_fields.end());
+        return arrow::schema(std::move(fields));
+    }
 };
 
 }  // namespace paimon
diff --git a/src/paimon/common/table/special_fields_test.cpp b/src/paimon/common/table/special_fields_test.cpp
index 58a025ba2..a0a0980a5 100644
--- a/src/paimon/common/table/special_fields_test.cpp
+++ b/src/paimon/common/table/special_fields_test.cpp
@@ -66,6 +66,27 @@ TEST(SpecialFieldsTest, TestKeyValueSpecialFieldCount) {
     ASSERT_EQ(SpecialFields::KEY_VALUE_SPECIAL_FIELD_COUNT, 2);
 }
 
+TEST(SpecialFieldsTest, TestPreparedKeyValueSchema) {
+    arrow::FieldVector value_fields = {arrow::field("key", arrow::int64(), false),
+                                       arrow::field("value", arrow::utf8())};
+    std::shared_ptr<arrow::Schema> schema = SpecialFields::PreparedKeyValueSchema(value_fields);
+
+    ASSERT_EQ(SpecialFields::kPreparedKeyValueValueKindIndex, 0);
+    ASSERT_EQ(SpecialFields::kPreparedKeyValueSequenceNumberIndex, 1);
+    ASSERT_EQ(SpecialFields::kPreparedKeyValueRealtimeOffsetIndex, 2);
+    ASSERT_EQ(SpecialFields::kPreparedKeyValueValueStartIndex, 3);
+    ASSERT_EQ(schema->field(0)->name(), "_VALUE_KIND");
+    ASSERT_EQ(schema->field(1)->name(), "_SEQUENCE_NUMBER");
+    ASSERT_EQ(schema->field(2)->name(), "_REALTIME_OFFSET");
+    ASSERT_EQ(schema->field(3)->name(), "key");
+    ASSERT_EQ(schema->field(4)->name(), "value");
+    ASSERT_FALSE(schema->field(0)->nullable());
+    ASSERT_FALSE(schema->field(1)->nullable());
+    ASSERT_EQ(schema->field(2)->nullable(), SpecialFields::RealtimeOffset().Nullable());
+    ASSERT_FALSE(schema->field(3)->nullable());
+    ASSERT_TRUE(schema->field(4)->nullable());
+}
+
 TEST(SpecialFieldsTest, TestIsSystemField) {
     ASSERT_TRUE(SpecialFields::IsSystemField("_SEQUENCE_NUMBER"));
     ASSERT_TRUE(SpecialFields::IsSystemField("_VALUE_KIND"));
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index 2a2251784..4a0a52920 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -48,11 +48,6 @@ namespace paimon {
 
 namespace {
 
-constexpr int32_t kValueKindIndex = 0;
-constexpr int32_t kSequenceNumberIndex = 1;
-constexpr int32_t kRealtimeOffsetIndex = 2;
-constexpr int32_t kPreparedValueStartIndex = 3;
-
 template <typename Reader>
 void CloseReaders(const std::vector<std::unique_ptr<Reader>>& readers) {
     for (const std::unique_ptr<Reader>& reader : readers) {
@@ -151,7 +146,7 @@ Result<std::vector<int32_t>> ResolveFieldIndexes(
     const std::shared_ptr<arrow::Schema>& prepared_schema,
     const std::shared_ptr<arrow::Schema>& row_schema) {
     arrow::FieldVector prepared_value_fields(
-        prepared_schema->fields().begin() + kPreparedValueStartIndex,
+        prepared_schema->fields().begin() + SpecialFields::kPreparedKeyValueValueStartIndex,
         prepared_schema->fields().end());
     std::vector<int32_t> result;
     result.reserve(row_schema->num_fields());
@@ -167,18 +162,19 @@ Result<std::vector<int32_t>> ResolveFieldIndexes(
                 "type {}",
                 field_id, prepared_field->type()->ToString(), row_field->type()->ToString()));
         }
-        result.push_back(value_index + kPreparedValueStartIndex);
+        result.push_back(value_index + SpecialFields::kPreparedKeyValueValueStartIndex);
     }
     return result;
 }
 
 Status ValidateExactCommitSchema(const std::shared_ptr<arrow::Schema>& prepared_schema,
                                  const std::shared_ptr<arrow::Schema>& value_schema) {
-    if (prepared_schema->num_fields() != value_schema->num_fields() + kPreparedValueStartIndex) {
+    if (prepared_schema->num_fields() !=
+        value_schema->num_fields() + SpecialFields::kPreparedKeyValueValueStartIndex) {
         return Status::Invalid("commit requires the exact prepared writer schema");
     }
     for (int32_t i = 0; i < value_schema->num_fields(); ++i) {
-        if (!prepared_schema->field(i + kPreparedValueStartIndex)
+        if (!prepared_schema->field(i + SpecialFields::kPreparedKeyValueValueStartIndex)
                  ->Equals(value_schema->field(i), true)) {
             return Status::Invalid("commit requires the exact prepared writer schema");
         }
@@ -300,15 +296,15 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
 
             std::shared_ptr<arrow::NumericArray<arrow::Int64Type>> offset_array =
                 checked_pointer_cast<arrow::NumericArray<arrow::Int64Type>>(
-                    data_batch->field(kRealtimeOffsetIndex));
+                    data_batch->field(SpecialFields::kPreparedKeyValueRealtimeOffsetIndex));
             if (offset_coverage_) {
                 PAIMON_RETURN_NOT_OK(offset_coverage_->Add(*offset_array));
             }
 
             row_kind_array_ = checked_pointer_cast<arrow::NumericArray<arrow::Int8Type>>(
-                data_batch->field(kValueKindIndex));
+                data_batch->field(SpecialFields::kPreparedKeyValueValueKindIndex));
             sequence_number_array_ = checked_pointer_cast<arrow::NumericArray<arrow::Int64Type>>(
-                data_batch->field(kSequenceNumberIndex));
+                data_batch->field(SpecialFields::kPreparedKeyValueSequenceNumberIndex));
             arrow::ArrayVector key_fields;
             key_fields.reserve(key_field_indexes_.size());
             for (int32_t index : key_field_indexes_) {
@@ -344,21 +340,26 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
                     "prepared batch field {} does not match declared prepared schema", i));
             }
         }
-        if (!data_batch->field(kValueKindIndex) ||
-            data_batch->field(kValueKindIndex)->type_id() != arrow::Type::INT8) {
+        if (!data_batch->field(SpecialFields::kPreparedKeyValueValueKindIndex) ||
+            data_batch->field(SpecialFields::kPreparedKeyValueValueKindIndex)->type_id() !=
+                arrow::Type::INT8) {
             return Status::Invalid("cannot cast VALUE_KIND column to int8 arrow array");
         }
-        if (!data_batch->field(kSequenceNumberIndex) ||
-            data_batch->field(kSequenceNumberIndex)->type_id() != arrow::Type::INT64) {
+        if (!data_batch->field(SpecialFields::kPreparedKeyValueSequenceNumberIndex) ||
+            data_batch->field(SpecialFields::kPreparedKeyValueSequenceNumberIndex)->type_id() !=
+                arrow::Type::INT64) {
             return Status::Invalid("cannot cast SEQUENCE_NUMBER column to int64 arrow array");
         }
-        if (!data_batch->field(kRealtimeOffsetIndex) ||
-            data_batch->field(kRealtimeOffsetIndex)->type_id() != arrow::Type::INT64) {
+        if (!data_batch->field(SpecialFields::kPreparedKeyValueRealtimeOffsetIndex) ||
+            data_batch->field(SpecialFields::kPreparedKeyValueRealtimeOffsetIndex)->type_id() !=
+                arrow::Type::INT64) {
             return Status::Invalid("cannot cast REALTIME_OFFSET column to int64 arrow array");
         }
-        if (data_batch->field(kValueKindIndex)->null_count() != 0 ||
-            data_batch->field(kSequenceNumberIndex)->null_count() != 0 ||
-            data_batch->field(kRealtimeOffsetIndex)->null_count() != 0) {
+        if (data_batch->field(SpecialFields::kPreparedKeyValueValueKindIndex)->null_count() != 0 ||
+            data_batch->field(SpecialFields::kPreparedKeyValueSequenceNumberIndex)->null_count() !=
+                0 ||
+            data_batch->field(SpecialFields::kPreparedKeyValueRealtimeOffsetIndex)->null_count() !=
+                0) {
             return Status::Invalid("prepared transport columns must not contain nulls");
         }
         return Status::OK();
@@ -428,15 +429,19 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
 
 Status PreparedKeyValueReaderFactory::ValidateTransportSchema(
     const std::shared_ptr<arrow::Schema>& prepared_schema) {
-    if (!prepared_schema || prepared_schema->num_fields() < kPreparedValueStartIndex) {
+    if (!prepared_schema ||
+        prepared_schema->num_fields() < SpecialFields::kPreparedKeyValueValueStartIndex) {
         return Status::Invalid("prepared schema must contain realtime transport fields");
     }
-    PAIMON_RETURN_NOT_OK(
-        CheckPreparedField(prepared_schema, kValueKindIndex, SpecialFields::ValueKind()));
-    PAIMON_RETURN_NOT_OK(
-        CheckPreparedField(prepared_schema, kSequenceNumberIndex, SpecialFields::SequenceNumber()));
-    PAIMON_RETURN_NOT_OK(
-        CheckPreparedField(prepared_schema, kRealtimeOffsetIndex, SpecialFields::RealtimeOffset()));
+    PAIMON_RETURN_NOT_OK(CheckPreparedField(prepared_schema,
+                                            SpecialFields::kPreparedKeyValueValueKindIndex,
+                                            SpecialFields::ValueKind()));
+    PAIMON_RETURN_NOT_OK(CheckPreparedField(prepared_schema,
+                                            SpecialFields::kPreparedKeyValueSequenceNumberIndex,
+                                            SpecialFields::SequenceNumber()));
+    PAIMON_RETURN_NOT_OK(CheckPreparedField(prepared_schema,
+                                            SpecialFields::kPreparedKeyValueRealtimeOffsetIndex,
+                                            SpecialFields::RealtimeOffset()));
     return Status::OK();
 }
 
@@ -474,9 +479,9 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderImpl(
                            ResolveFieldIndexes(prepared_schema, key_schema));
     PAIMON_ASSIGN_OR_RAISE(std::vector<int32_t> value_field_indexes,
                            ResolveFieldIndexes(prepared_schema, value_schema));
-    std::unique_ptr<KeyValueRecordReader> result(new PreparedKeyValueReader(
+    std::unique_ptr<KeyValueRecordReader> result = std::make_unique<PreparedKeyValueReader>(
         std::move(owned_reader), prepared_schema, visible_offsets, std::move(key_field_indexes),
-        std::move(value_field_indexes), memory_pool, offset_coverage));
+        std::move(value_field_indexes), memory_pool, offset_coverage);
     close_guard.Release();
     return result;
 }
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index 89e86e787..ae123a643 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -27,7 +27,6 @@
 #include "arrow/c/bridge.h"
 #include "arrow/compute/api.h"
 #include "paimon/common/table/special_fields.h"
-#include "paimon/common/types/data_field.h"
 #include "paimon/common/types/row_kind.h"
 #include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
@@ -139,22 +138,16 @@ Result<std::shared_ptr<RealtimePrimaryKeyWriter>> RealtimePrimaryKeyWriter::Crea
         }
         key_fields.push_back(std::move(field));
     }
-    arrow::FieldVector prepared_fields = {
-        DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
-        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
-            ->WithNullable(false),
-        DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset())};
-    prepared_fields.insert(prepared_fields.end(), write_schema->fields().begin(),
-                           write_schema->fields().end());
+    std::shared_ptr<arrow::Schema> prepared_schema =
+        SpecialFields::PreparedKeyValueSchema(write_schema->fields());
     const RealtimePartitionBucket partition_bucket(partition, bucket);
     PAIMON_ASSIGN_OR_RAISE(int64_t initial_max_sequence_number,
                            realtime_context->AdvanceMaterializedMaxSequenceNumber(
                                partition_bucket, restored_max_sequence_number));
     return std::shared_ptr<RealtimePrimaryKeyWriter>(new RealtimePrimaryKeyWriter(
         store_state.store, merge_tree_writer, realtime_context, partition_bucket, write_schema,
-        arrow::schema(std::move(prepared_fields)), arrow::schema(std::move(key_fields)),
-        trimmed_primary_keys, key_comparator, store_state.initial_offset,
-        initial_max_sequence_number, memory_pool));
+        prepared_schema, arrow::schema(std::move(key_fields)), trimmed_primary_keys, key_comparator,
+        store_state.initial_offset, initial_max_sequence_number, memory_pool));
 }
 
 RealtimePrimaryKeyWriter::RealtimePrimaryKeyWriter(
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index e13140f54..e4225587d 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -19,6 +19,7 @@
 
 #include "paimon/core/table/source/key_value_table_read.h"
 
+#include <unordered_set>
 #include <utility>
 #include <vector>
 
@@ -26,7 +27,6 @@
 #include "arrow/c/bridge.h"
 #include "paimon/common/reader/concat_batch_reader.h"
 #include "paimon/common/table/special_fields.h"
-#include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/global_index/indexed_split_impl.h"
@@ -42,6 +42,7 @@
 #include "paimon/core/table/source/data_split_impl.h"
 #include "paimon/core/table/source/pk_count_reader.h"
 #include "paimon/core/table/source/realtime_split.h"
+#include "paimon/core/utils/nested_projection_utils.h"
 #include "paimon/core/utils/primary_key_table_utils.h"
 #include "paimon/status.h"
 
@@ -62,14 +63,23 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateMemoryReaders(
     const std::shared_ptr<FieldsComparator>& key_comparator,
     const std::shared_ptr<InternalReadContext>& context,
     const std::shared_ptr<MemoryPool>& memory_pool) {
-    arrow::FieldVector prepared_fields = {
-        DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
-        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
-            ->WithNullable(false),
-        DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset())};
-    prepared_fields.insert(prepared_fields.end(), value_schema->fields().begin(),
-                           value_schema->fields().end());
-    std::shared_ptr<arrow::Schema> prepared_schema = arrow::schema(std::move(prepared_fields));
+    arrow::FieldVector prepared_value_fields;
+    prepared_value_fields.reserve(key_schema->num_fields() + value_schema->num_fields());
+    std::unordered_set<int32_t> field_ids;
+    for (const std::shared_ptr<arrow::Field>& field : key_schema->fields()) {
+        PAIMON_ASSIGN_OR_RAISE(int32_t field_id, NestedProjectionUtils::GetPaimonFieldId(field));
+        if (field_ids.insert(field_id).second) {
+            prepared_value_fields.push_back(field);
+        }
+    }
+    for (const std::shared_ptr<arrow::Field>& field : value_schema->fields()) {
+        PAIMON_ASSIGN_OR_RAISE(int32_t field_id, NestedProjectionUtils::GetPaimonFieldId(field));
+        if (field_ids.insert(field_id).second) {
+            prepared_value_fields.push_back(field);
+        }
+    }
+    std::shared_ptr<arrow::Schema> prepared_schema =
+        SpecialFields::PreparedKeyValueSchema(prepared_value_fields);
     auto c_schema = std::make_unique<ArrowSchema>();
     PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*prepared_schema, c_schema.get()));
     ScopeGuard schema_guard([schema = c_schema.get()]() { ArrowSchemaRelease(schema); });
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index 62db90296..73ddf050b 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -1882,6 +1882,60 @@ TEST_F(RealtimeWriteInteTest, TestPkNestedProjectionAcrossDiskAndMemory) {
     ASSERT_OK(writer->Close());
 }
 
+TEST_F(RealtimeWriteInteTest, TestPkKeylessProjection) {
+    CreatePkTable();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
+                         MakeBatch({Row{1, "disk", "p0"}}, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(disk_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_progress,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(disk_progress, /*commit_identifier=*/0));
+    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> memory_batch,
+                         MakeBatch({Row{1, "memory", "p0"}}, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(memory_batch)));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
+                         CreatePlan(realtime_context, /*predicate=*/nullptr));
+    ReadPlanWithSchemaAndCheck(plan, realtime_context,
+                               arrow::schema({arrow::field("payload", arrow::utf8())}), R"([
+        [0, "memory"]
+    ])");
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestCompositePkKeylessProjection) {
+    CreatePkTable(/*partition_keys=*/{}, /*primary_keys=*/{"id", "payload"});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
+                         MakeBatch({Row{1, "key", "disk"}}, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(disk_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_progress,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(disk_progress, /*commit_identifier=*/0));
+    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> memory_batch,
+                         MakeBatch({Row{1, "key", "memory"}}, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(memory_batch)));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
+                         CreatePlan(realtime_context, /*predicate=*/nullptr));
+    ReadPlanWithSchemaAndCheck(plan, realtime_context,
+                               arrow::schema({arrow::field("pt", arrow::utf8())}), R"([
+        [0, "memory"]
+    ])");
+    ASSERT_OK(writer->Close());
+}
+
 TEST_F(RealtimeWriteInteTest, TestPkCompositeMerge) {
     CreatePkTable(/*partition_keys=*/{}, /*primary_keys=*/{"id", "payload"});
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,

From 09c53c905c70946faf02f628cb562280624b75e0 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Wed, 26 Aug 2026 18:41:27 +0800
Subject: [PATCH 44/62] fix(read): bound realtime merge fan-in

---
 .../core/operation/merge_file_split_read.cpp  | 104 ++++++++++++++++--
 .../core/operation/merge_file_split_read.h    |  10 ++
 .../operation/merge_file_split_read_test.cpp  |  85 +++++++++++++-
 3 files changed, 184 insertions(+), 15 deletions(-)

diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp
index 02a7fac20..6c8bbecad 100644
--- a/src/paimon/core/operation/merge_file_split_read.cpp
+++ b/src/paimon/core/operation/merge_file_split_read.cpp
@@ -36,6 +36,7 @@
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/object_utils.h"
+#include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/deletionvectors/apply_deletion_vector_batch_reader.h"
 #include "paimon/core/deletionvectors/bitmap_deletion_vector.h"
@@ -78,6 +79,53 @@ struct KeyValue;
 template <typename T>
 class MergeFunctionWrapper;
 
+namespace {
+
+class SortMergeKeyValueRecordReader : public KeyValueRecordReader {
+ public:
+    explicit SortMergeKeyValueRecordReader(std::unique_ptr<SortMergeReader>&& reader)
+        : reader_(std::move(reader)) {}
+
+    class Iterator : public KeyValueRecordReader::Iterator {
+     public:
+        explicit Iterator(std::unique_ptr<SortMergeReader::Iterator>&& iterator)
+            : iterator_(std::move(iterator)) {}
+
+        Result<bool> HasNext() const override {
+            return iterator_->HasNext();
+        }
+
+        Result<KeyValue> Next() override {
+            return std::move(iterator_->Next());
+        }
+
+     private:
+        std::unique_ptr<SortMergeReader::Iterator> iterator_;
+    };
+
+    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatch() override {
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<SortMergeReader::Iterator> iterator,
+                               reader_->NextBatch());
+        if (!iterator) {
+            return std::unique_ptr<KeyValueRecordReader::Iterator>();
+        }
+        return std::make_unique<Iterator>(std::move(iterator));
+    }
+
+    void Close() override {
+        reader_->Close();
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return reader_->GetReaderMetrics();
+    }
+
+ private:
+    std::unique_ptr<SortMergeReader> reader_;
+};
+
+}  // namespace
+
 class MergeFileSplitRead::RealtimeReaderBuilder {
  public:
     static Result<std::unique_ptr<BatchReader>> Create(
@@ -145,15 +193,34 @@ class MergeFileSplitRead::RealtimeReaderBuilder {
         std::vector<std::vector<SortedRun>> disk_sections;
         PAIMON_RETURN_NOT_OK(
             owner_->CreateDiskSections(data_files, deletion_files, &dv_factory, &disk_sections));
+        std::vector<std::unique_ptr<KeyValueRecordReader>> section_readers;
+        ScopeGuard section_readers_guard([&section_readers]() {
+            for (const std::unique_ptr<KeyValueRecordReader>& reader : section_readers) {
+                reader->Close();
+            }
+        });
+        section_readers.reserve(disk_sections.size());
+        std::shared_ptr<MergeFunctionWrapper<KeyValue>> merge_function_wrapper;
+        if (!disk_sections.empty()) {
+            PAIMON_ASSIGN_OR_RAISE(merge_function_wrapper,
+                                   MergeFileSplitRead::CreateMergeFunctionWrapper(
+                                       owner_->options_, owner_->context_->GetTableSchema(),
+                                       owner_->value_schema_, owner_->pool_));
+        }
         for (const std::vector<SortedRun>& section : disk_sections) {
             PAIMON_ASSIGN_OR_RAISE(
-                std::vector<std::unique_ptr<KeyValueRecordReader>> section_readers,
-                owner_->CreateRecordReadersForSection(section, partition, dv_factory,
-                                                      owner_->predicate_for_keys_,
-                                                      data_file_path_factory));
-            for (std::unique_ptr<KeyValueRecordReader>& reader : section_readers) {
-                readers->push_back(std::move(reader));
-            }
+                std::unique_ptr<SortMergeReader> section_reader,
+                owner_->CreateSortMergeReaderForSection(
+                    section, partition, dv_factory, owner_->predicate_for_keys_,
+                    data_file_path_factory, /*drop_delete=*/false, merge_function_wrapper));
+            section_readers.push_back(
+                std::make_unique<SortMergeKeyValueRecordReader>(std::move(section_reader)));
+        }
+        if (!section_readers.empty()) {
+            std::unique_ptr<KeyValueRecordReader> concat_reader =
+                std::make_unique<ConcatKeyValueRecordReader>(std::move(section_readers));
+            section_readers_guard.Release();
+            readers->push_back(std::move(concat_reader));
         }
         return Status::OK();
     }
@@ -618,11 +685,24 @@ Result<std::unique_ptr<SortMergeReader>> MergeFileSplitRead::CreateSortMergeRead
     const std::vector<SortedRun>& section, const BinaryRow& partition,
     DeletionVector::Factory dv_factory, const std::shared_ptr<Predicate>& predicate,
     const std::shared_ptr<DataFilePathFactory>& data_file_path_factory, bool drop_delete) {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<MergeFunctionWrapper<KeyValue>> merge_function_wrapper,
+                           GetMergeFunctionWrapper());
+    return CreateSortMergeReaderForSection(section, partition, dv_factory, predicate,
+                                           data_file_path_factory, drop_delete,
+                                           merge_function_wrapper);
+}
+
+Result<std::unique_ptr<SortMergeReader>> MergeFileSplitRead::CreateSortMergeReaderForSection(
+    const std::vector<SortedRun>& section, const BinaryRow& partition,
+    DeletionVector::Factory dv_factory, const std::shared_ptr<Predicate>& predicate,
+    const std::shared_ptr<DataFilePathFactory>& data_file_path_factory, bool drop_delete,
+    const std::shared_ptr<MergeFunctionWrapper<KeyValue>>& merge_function_wrapper) {
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> record_readers,
                            CreateRecordReadersForSection(section, partition, dv_factory, predicate,
                                                          data_file_path_factory));
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<SortMergeReader> sort_merge_reader,
-                           CreateSortMergeReader(std::move(record_readers)));
+    PAIMON_ASSIGN_OR_RAISE(
+        std::unique_ptr<SortMergeReader> sort_merge_reader,
+        CreateSortMergeReader(std::move(record_readers), merge_function_wrapper));
     if (drop_delete) {
         sort_merge_reader = std::make_unique<DropDeleteReader>(std::move(sort_merge_reader));
     }
@@ -657,6 +737,12 @@ Result<std::unique_ptr<SortMergeReader>> MergeFileSplitRead::CreateSortMergeRead
     std::vector<std::unique_ptr<KeyValueRecordReader>>&& record_readers) {
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<MergeFunctionWrapper<KeyValue>> merge_function_wrapper,
                            GetMergeFunctionWrapper());
+    return CreateSortMergeReader(std::move(record_readers), merge_function_wrapper);
+}
+
+Result<std::unique_ptr<SortMergeReader>> MergeFileSplitRead::CreateSortMergeReader(
+    std::vector<std::unique_ptr<KeyValueRecordReader>>&& record_readers,
+    const std::shared_ptr<MergeFunctionWrapper<KeyValue>>& merge_function_wrapper) const {
     auto sort_engine = options_.GetSortEngine();
     if (sort_engine == SortEngine::MIN_HEAP) {
         return std::make_unique<SortMergeReaderWithMinHeap>(
diff --git a/src/paimon/core/operation/merge_file_split_read.h b/src/paimon/core/operation/merge_file_split_read.h
index ad45e4cf5..b54c96331 100644
--- a/src/paimon/core/operation/merge_file_split_read.h
+++ b/src/paimon/core/operation/merge_file_split_read.h
@@ -157,6 +157,12 @@ class MergeFileSplitRead : public AbstractSplitRead {
         std::unique_ptr<SortMergeReader>&& sort_merge_reader,
         const std::shared_ptr<Predicate>& predicate, bool complete_row_kind);
 
+    Result<std::unique_ptr<SortMergeReader>> CreateSortMergeReaderForSection(
+        const std::vector<SortedRun>& section, const BinaryRow& partition,
+        DeletionVector::Factory dv_factory, const std::shared_ptr<Predicate>& predicate,
+        const std::shared_ptr<DataFilePathFactory>& data_file_path_factory, bool drop_delete,
+        const std::shared_ptr<MergeFunctionWrapper<KeyValue>>& merge_function_wrapper);
+
     Result<std::unique_ptr<KeyValueRecordReader>> CreateReaderForRun(
         const BinaryRow& partition, const SortedRun& sorted_run, DeletionVector::Factory dv_factory,
         const std::shared_ptr<Predicate>& predicate,
@@ -165,6 +171,10 @@ class MergeFileSplitRead : public AbstractSplitRead {
     Result<std::unique_ptr<SortMergeReader>> CreateSortMergeReader(
         std::vector<std::unique_ptr<KeyValueRecordReader>>&& record_readers);
 
+    Result<std::unique_ptr<SortMergeReader>> CreateSortMergeReader(
+        std::vector<std::unique_ptr<KeyValueRecordReader>>&& record_readers,
+        const std::shared_ptr<MergeFunctionWrapper<KeyValue>>& merge_function_wrapper) const;
+
     Result<std::shared_ptr<MergeFunctionWrapper<KeyValue>>> GetMergeFunctionWrapper();
 
     MergeFileSplitRead(const std::shared_ptr<FileStorePathFactory>& path_factory,
diff --git a/src/paimon/core/operation/merge_file_split_read_test.cpp b/src/paimon/core/operation/merge_file_split_read_test.cpp
index 911cf7961..a899fd904 100644
--- a/src/paimon/core/operation/merge_file_split_read_test.cpp
+++ b/src/paimon/core/operation/merge_file_split_read_test.cpp
@@ -40,6 +40,7 @@
 #include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/io/key_value_in_memory_record_reader.h"
 #include "paimon/core/manifest/file_source.h"
 #include "paimon/core/operation/internal_read_context.h"
 #include "paimon/core/schema/schema_manager.h"
@@ -328,9 +329,8 @@ class MergeFileSplitReadTest : public ::testing::Test,
         return {data_split1};
     }
 
-    Result<std::unique_ptr<BatchReader>> CreateReader(
-        const std::shared_ptr<InternalReadContext>& internal_context,
-        const std::vector<std::shared_ptr<DataSplit>>& data_splits) {
+    Result<std::unique_ptr<MergeFileSplitRead>> CreateMergeFileSplitRead(
+        const std::shared_ptr<InternalReadContext>& internal_context) {
         const auto& core_options = internal_context->GetCoreOptions();
         const auto& table_schema = internal_context->GetTableSchema();
         auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields());
@@ -347,9 +347,14 @@ class MergeFileSplitReadTest : public ::testing::Test,
                 core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(),
                 external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(),
                 pool_));
-        PAIMON_ASSIGN_OR_RAISE(auto split_read,
-                               MergeFileSplitRead::Create(path_factory, std::move(internal_context),
-                                                          pool_, executor_));
+        return MergeFileSplitRead::Create(path_factory, internal_context, pool_, executor_);
+    }
+
+    Result<std::unique_ptr<BatchReader>> CreateReader(
+        const std::shared_ptr<InternalReadContext>& internal_context,
+        const std::vector<std::shared_ptr<DataSplit>>& data_splits) {
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<MergeFileSplitRead> split_read,
+                               CreateMergeFileSplitRead(internal_context));
         std::vector<std::unique_ptr<BatchReader>> batch_readers;
         batch_readers.reserve(data_splits.size());
         for (const auto& split : data_splits) {
@@ -666,6 +671,74 @@ TEST_P(MergeFileSplitReadTest, TestSimple) {
     CheckResult(result_array, expected_array, read_schema);
 }
 
+TEST_P(MergeFileSplitReadTest, TestRealtimeReadConcatenatesOrderedDiskSections) {
+    std::string path =
+        paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor";
+    ReadContextBuilder context_builder(path);
+    std::vector<DataField> raw_read_fields = {DataField(0, arrow::field("k0", arrow::int32())),
+                                              DataField(1, arrow::field("k1", arrow::int32())),
+                                              DataField(5, arrow::field("s1", arrow::utf8())),
+                                              DataField(6, arrow::field("v0", arrow::float64()))};
+    std::shared_ptr<arrow::Schema> read_schema =
+        DataField::ConvertDataFieldsToArrowSchema(raw_read_fields);
+    ASSERT_TRUE(read_schema);
+
+    context_builder.SetReadFieldNames({"k0", "k1", "s1", "v0"});
+    context_builder.SetOptions(
+        {{Options::SEQUENCE_FIELD, "s0,s1"}, {Options::MERGE_ENGINE, "deduplicate"}});
+    AddOptions(&context_builder);
+    context_builder.EnableMultiThreadRowToBatch(false);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<ReadContext> read_context, context_builder.Finish());
+    std::shared_ptr<InternalReadContext> internal_context = CreateInternalReadContext(read_context);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<MergeFileSplitRead> split_read,
+                         CreateMergeFileSplitRead(internal_context));
+
+    std::shared_ptr<arrow::DataType> memory_type =
+        arrow::struct_(split_read->GetValueSchema()->fields());
+    std::shared_ptr<arrow::StructArray> memory_array =
+        std::dynamic_pointer_cast<arrow::StructArray>(
+            arrow::ipc::internal::json::ArrayFromJSON(memory_type, R"([
+                [100, 200, "memory-late",   10000.0, "zzzz"],
+                [1,     1, "memory-delete",  1100.0, "zzzz"],
+                [0,     0, "memory-first",   1000.0, "zzzz"],
+                [50,    0, "memory-middle",  5000.0, "zzzz"]
+            ])")
+                .ValueOrDie());
+    std::vector<std::unique_ptr<KeyValueRecordReader>> memory_readers;
+    memory_readers.push_back(std::make_unique<KeyValueInMemoryRecordReader>(
+        /*last_sequence_num=*/9, memory_array,
+        std::vector<RecordBatch::RowKind>(
+            {RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::DELETE,
+             RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::INSERT}),
+        std::vector<std::string>({"k0", "k1"}), std::vector<std::string>({"s0", "s1"}),
+        /*sequence_fields_ascending=*/true, split_read->GetKeyComparator(), pool_));
+
+    std::vector<std::shared_ptr<Split>> disk_splits = {PrepareDataSplit().front()};
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> batch_reader,
+                         split_read->CreateRealtimeReader(disk_splits, std::move(memory_readers)));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> result_array,
+                         ReadResultCollector::CollectResult(batch_reader.get()));
+
+    arrow::FieldVector fields_with_row_kind = read_schema->fields();
+    fields_with_row_kind.insert(fields_with_row_kind.begin(),
+                                arrow::field("_VALUE_KIND", arrow::int8()));
+    std::shared_ptr<arrow::ChunkedArray> expected_array;
+    auto expected_status =
+        arrow::ipc::internal::json::ChunkedArrayFromJSON(arrow::struct_(fields_with_row_kind), {R"([
+            [0,   0,   0, "memory-first",  1000.0],
+            [0,   0,   1, "you",             11.1],
+            [0,   1,   0, "later",           12.2],
+            [0,   1,   2, "!",               13.3],
+            [0,  50,   0, "memory-middle", 5000.0],
+            [0, 100, 200, "memory-late", 10000.0]
+        ])"},
+                                                         &expected_array);
+    ASSERT_TRUE(expected_status.ok());
+    CheckResult(result_array, expected_array, read_schema);
+    ASSERT_TRUE(batch_reader->GetReaderMetrics());
+    batch_reader->Close();
+}
+
 TEST_P(MergeFileSplitReadTest, TestLookUp) {
     std::string path =
         paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor";

From 712b1e7b6e4806666c3b5bac01d513f5b3039968 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Wed, 26 Aug 2026 18:41:57 +0800
Subject: [PATCH 45/62] refactor(realtime): simplify primary key maintenance

---
 .../core/operation/file_store_write.cpp       |  5 ++-
 .../operation/key_value_file_store_write.cpp  | 19 ++++-------
 .../realtime/primary_key_realtime_store.cpp   | 11 ++++---
 .../primary_key_realtime_store_test.cpp       | 32 +++++++++++++++++--
 .../core/realtime/realtime_context_impl.cpp   | 15 +++++----
 .../realtime/realtime_primary_key_writer.h    |  1 +
 .../core/utils/primary_key_table_utils.cpp    | 22 ++++++-------
 .../utils/primary_key_table_utils_test.cpp    |  8 ++---
 8 files changed, 68 insertions(+), 45 deletions(-)

diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp
index 9710b57f0..fa1294d1b 100644
--- a/src/paimon/core/operation/file_store_write.cpp
+++ b/src/paimon/core/operation/file_store_write.cpp
@@ -212,11 +212,10 @@ Result<std::unique_ptr<FileStoreWrite>> FileStoreWrite::Create(std::unique_ptr<W
             PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *schema));
             if (ignore_previous_files) {
                 return Status::NotImplemented(
-                    "PK realtime v1 requires restore from the latest snapshot");
+                    "PK realtime requires restore from the latest snapshot");
             }
             if (!ctx->GetWriteSchema().empty()) {
-                return Status::NotImplemented(
-                    "PK realtime v1 does not support a custom write schema");
+                return Status::NotImplemented("PK realtime does not support a custom write schema");
             }
             PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx->GetRealtimeContext(),
                                                                   snapshot_manager, options));
diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp
index c6a62c107..f431597a7 100644
--- a/src/paimon/core/operation/key_value_file_store_write.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write.cpp
@@ -24,7 +24,6 @@
 #include "arrow/c/bridge.h"
 #include "paimon/common/data/binary_row.h"
 #include "paimon/common/table/special_fields.h"
-#include "paimon/common/types/data_field.h"
 #include "paimon/core/compact/noop_compact_manager.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/io/data_file_meta.h"
@@ -121,9 +120,6 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
                            file_store_path_factory_->CreateDataFilePathFactory(partition, bucket));
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::string> trimmed_primary_keys,
                            table_schema_->TrimmedPrimaryKeys());
-    PAIMON_ASSIGN_OR_RAISE(
-        std::shared_ptr<Levels> levels,
-        Levels::Create(key_comparator_, restore_data_files, options_.GetNumLevels()));
     std::map<std::string, std::string> partition_map;
     std::shared_ptr<CompactManager> compact_manager;
     std::shared_ptr<RealtimeContextImpl> realtime_context_impl;
@@ -139,17 +135,11 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
             return Status::Invalid("PK real-time write schema contains reserved transport field " +
                                    SpecialFields::RealtimeOffset().Name());
         }
-        arrow::FieldVector prepared_fields = {
-            DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())
-                ->WithNullable(false),
-            DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
-                ->WithNullable(false),
-            DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset())};
-        prepared_fields.insert(prepared_fields.end(), schema_->fields().begin(),
-                               schema_->fields().end());
+        std::shared_ptr<arrow::Schema> prepared_schema =
+            SpecialFields::PreparedKeyValueSchema(schema_->fields());
         auto c_write_schema = std::make_unique<ArrowSchema>();
         PAIMON_RETURN_NOT_OK_FROM_ARROW(
-            arrow::ExportSchema(*arrow::schema(std::move(prepared_fields)), c_write_schema.get()));
+            arrow::ExportSchema(*prepared_schema, c_write_schema.get()));
         PAIMON_ASSIGN_OR_RAISE(
             RealtimeStoreState store_state,
             realtime_context_impl->GetOrCreateRealtimeStore(
@@ -159,6 +149,9 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
         realtime_store_state = std::move(store_state);
         compact_manager = std::make_shared<NoopCompactManager>();
     } else {
+        PAIMON_ASSIGN_OR_RAISE(
+            std::shared_ptr<Levels> levels,
+            Levels::Create(key_comparator_, restore_data_files, options_.GetNumLevels()));
         auto compact_strategy = compact_manager_factory_->CreateCompactStrategy();
         PAIMON_ASSIGN_OR_RAISE(compact_manager, compact_manager_factory_->CreateCompactManager(
                                                     partition, bucket, compact_strategy,
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index 847347113..dc3498424 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -18,6 +18,7 @@
 
 #include "paimon/core/realtime/primary_key_realtime_store.h"
 
+#include <algorithm>
 #include <mutex>
 #include <optional>
 #include <utility>
@@ -204,7 +205,7 @@ class PrimaryKeyRealtimeStore::Impl {
             segments.push_back(
                 std::make_shared<Segment>(range, std::vector<StoredBatch>(building_)));
         }
-        return std::shared_ptr<RealtimeReadView>(new ReadView(std::move(segments)));
+        return std::make_shared<ReadView>(std::move(segments));
     }
 
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
@@ -241,9 +242,11 @@ class PrimaryKeyRealtimeStore::Impl {
 
     Status AdvanceCommittedOffset(int64_t committed_end_offset) {
         std::lock_guard<std::mutex> lock(mutex_);
-        while (!sealed_.empty() && sealed_.front()->GetOffsetRange().end <= committed_end_offset) {
-            sealed_.erase(sealed_.begin());
-        }
+        auto first_retained = std::find_if(
+            sealed_.begin(), sealed_.end(), [committed_end_offset](const auto& segment) {
+                return segment->GetOffsetRange().end > committed_end_offset;
+            });
+        sealed_.erase(sealed_.begin(), first_retained);
         return Status::OK();
     }
 
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index 209d2ae7a..46d9a8e7d 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -319,19 +319,45 @@ TEST(PrimaryKeyRealtimeStoreTest, TestReclaimKeepsReadView) {
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
                          store->SealForCommit());
     ASSERT_TRUE(segment.has_value());
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
-    ASSERT_GT(store->GetMemoryUsage(), 0);
+    ASSERT_OK(
+        store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 1, 5, 2, "two"]])"), OffsetRange(5, 6)}));
+    ASSERT_OK_AND_ASSIGN(segment, store->SealForCommit());
+    ASSERT_TRUE(segment.has_value());
+    ASSERT_OK(store->Write(
+        RealtimeWriteBatch{MakeBatch(R"([[0, 2, 6, 3, "three"]])"), OffsetRange(6, 7)}));
+    ASSERT_OK_AND_ASSIGN(segment, store->SealForCommit());
+    ASSERT_TRUE(segment.has_value());
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> retained_view, store->AcquireReadView());
+    ASSERT_EQ(OffsetRange(4, 7), retained_view->GetOffsetRange());
+
+    const uint64_t initial_memory_usage = store->GetMemoryUsage();
+    ASSERT_GT(initial_memory_usage, 0);
+    ASSERT_OK(store->AdvanceCommittedOffset(4));
+    ASSERT_EQ(initial_memory_usage, store->GetMemoryUsage());
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> current_view, store->AcquireReadView());
+    ASSERT_EQ(OffsetRange(4, 7), current_view->GetOffsetRange());
     ASSERT_OK(store->AdvanceCommittedOffset(5));
+    ASSERT_LT(store->GetMemoryUsage(), initial_memory_usage);
+    ASSERT_OK_AND_ASSIGN(current_view, store->AcquireReadView());
+    ASSERT_EQ(OffsetRange(5, 7), current_view->GetOffsetRange());
+    ASSERT_OK(store->AdvanceCommittedOffset(6));
+    ASSERT_OK_AND_ASSIGN(current_view, store->AcquireReadView());
+    ASSERT_EQ(OffsetRange(6, 7), current_view->GetOffsetRange());
+    ASSERT_OK(store->AdvanceCommittedOffset(7));
     ASSERT_EQ(0, store->GetMemoryUsage());
+    ASSERT_OK_AND_ASSIGN(current_view, store->AcquireReadView());
+    ASSERT_FALSE(current_view->GetOffsetRange().has_value());
 
     auto c_schema = std::make_unique<ArrowSchema>();
     ASSERT_TRUE(arrow::ExportSchema(*PreparedSchema(), c_schema.get()).ok());
     RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
                                  /*enable_predicate_pushdown=*/false};
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
-                         store->CreateQueryReaders(view, /*offset_begin=*/0, context));
+                         store->CreateQueryReaders(retained_view, /*offset_begin=*/0, context));
     ASSERT_OK_AND_ASSIGN(std::string actual, ReadJson(readers));
     ASSERT_NE(std::string::npos, actual.find("\"one\""));
+    ASSERT_NE(std::string::npos, actual.find("\"two\""));
+    ASSERT_NE(std::string::npos, actual.find("\"three\""));
 }
 
 TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderPerStoredBatch) {
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index ad17ee695..2b9d7d07c 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -37,6 +37,7 @@
 #include "arrow/api.h"
 #include "arrow/c/bridge.h"
 #include "arrow/c/helpers.h"
+#include "fmt/format.h"
 #include "paimon/arrow/abi.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/scope_guard.h"
@@ -119,10 +120,10 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     if (iter != stores_.end()) {
         if (iter->second.mode != request.mode ||
             !iter->second.write_schema->Equals(*requested_schema, /*check_metadata=*/true)) {
-            return Status::Invalid("real-time store schema or mode mismatch for partition " +
-                                   PartitionToString(partition_bucket.partition) + ", bucket " +
-                                   std::to_string(partition_bucket.bucket) +
-                                   "; recreate the RealtimeContext");
+            return Status::Invalid(fmt::format(
+                "real-time store schema or mode mismatch for partition {}, bucket {}; recreate "
+                "the RealtimeContext",
+                PartitionToString(partition_bucket.partition), partition_bucket.bucket));
         }
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeReadView> read_view,
                                iter->second.store->AcquireReadView());
@@ -165,9 +166,9 @@ Result<int64_t> RealtimeContextImpl::AdvanceMaterializedMaxSequenceNumber(
     std::lock_guard<std::mutex> lock(mutex_);
     auto iter = stores_.find(partition_bucket);
     if (iter == stores_.end()) {
-        return Status::KeyError("real-time store not found for partition " +
-                                PartitionToString(partition_bucket.partition) + ", bucket " +
-                                std::to_string(partition_bucket.bucket));
+        return Status::KeyError(fmt::format("real-time store not found for partition {}, bucket {}",
+                                            PartitionToString(partition_bucket.partition),
+                                            partition_bucket.bucket));
     }
     StoreEntry& entry = iter->second;
     if (max_sequence_number > entry.materialized_max_sequence_number) {
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.h b/src/paimon/core/realtime/realtime_primary_key_writer.h
index d65c7e533..5cae9ca47 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.h
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.h
@@ -20,6 +20,7 @@
 #pragma once
 
 #include <cstdint>
+#include <map>
 #include <memory>
 #include <mutex>
 #include <string>
diff --git a/src/paimon/core/utils/primary_key_table_utils.cpp b/src/paimon/core/utils/primary_key_table_utils.cpp
index b5efb5ded..2ca444a7a 100644
--- a/src/paimon/core/utils/primary_key_table_utils.cpp
+++ b/src/paimon/core/utils/primary_key_table_utils.cpp
@@ -101,46 +101,46 @@ Result<std::unique_ptr<FieldsComparator>> PrimaryKeyTableUtils::CreateSequenceFi
 Status PrimaryKeyTableUtils::ValidateRealtimeOptions(const CoreOptions& options,
                                                      const TableSchema& schema) {
     if (options.GetBucket() <= 0) {
-        return Status::NotImplemented("PK realtime v1 requires fixed buckets");
+        return Status::NotImplemented("PK realtime requires fixed buckets");
     }
     if (options.GetMergeEngine() != MergeEngine::DEDUPLICATE) {
-        return Status::NotImplemented("PK realtime v1 supports only the DEDUPLICATE merge engine");
+        return Status::NotImplemented("PK realtime supports only the DEDUPLICATE merge engine");
     }
     if (options.DataEvolutionEnabled()) {
-        return Status::NotImplemented("PK realtime v1 does not support data evolution");
+        return Status::NotImplemented("PK realtime does not support data evolution");
     }
     if (options.IgnoreDelete()) {
-        return Status::NotImplemented("PK realtime v1 requires default delete behavior");
+        return Status::NotImplemented("PK realtime requires default delete behavior");
     }
     if (!options.GetSequenceField().empty()) {
-        return Status::NotImplemented("PK realtime v1 does not support sequence.field");
+        return Status::NotImplemented("PK realtime does not support sequence.field");
     }
     if (!options.SequenceFieldSortOrderIsAscending()) {
         return Status::NotImplemented(
-            "PK realtime v1 supports only ascending sequence.field.sort-order");
+            "PK realtime supports only ascending sequence.field.sort-order");
     }
     if (options.GetChangelogProducer() != ChangelogProducer::NONE) {
-        return Status::NotImplemented("PK realtime v1 supports only the NONE changelog producer");
+        return Status::NotImplemented("PK realtime supports only the NONE changelog producer");
     }
     if (options.DeletionVectorsEnabled()) {
-        return Status::NotImplemented("PK realtime v1 does not support deletion vectors");
+        return Status::NotImplemented("PK realtime does not support deletion vectors");
     }
     if (options.NeedLookup()) {
-        return Status::NotImplemented("PK realtime v1 does not support lookup");
+        return Status::NotImplemented("PK realtime does not support lookup");
     }
     PAIMON_ASSIGN_OR_RAISE(std::vector<DataField> primary_key_fields,
                            schema.TrimmedPrimaryKeyFields());
     for (const DataField& field : primary_key_fields) {
         if (field.Type()->id() == arrow::Type::FLOAT || field.Type()->id() == arrow::Type::DOUBLE) {
             return Status::NotImplemented(
-                "PK realtime v1 does not support FLOAT or DOUBLE primary keys");
+                "PK realtime does not support FLOAT or DOUBLE primary keys");
         }
     }
     if (options.GlobalIndexEnabled()) {
         PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions definitions,
                                PrimaryKeyIndexDefinitions::Create(schema));
         if (!definitions.Definitions().empty()) {
-            return Status::NotImplemented("PK realtime v1 does not support global indexes");
+            return Status::NotImplemented("PK realtime does not support global indexes");
         }
     }
     return Status::OK();
diff --git a/src/paimon/core/utils/primary_key_table_utils_test.cpp b/src/paimon/core/utils/primary_key_table_utils_test.cpp
index 8887102cd..796922336 100644
--- a/src/paimon/core/utils/primary_key_table_utils_test.cpp
+++ b/src/paimon/core/utils/primary_key_table_utils_test.cpp
@@ -113,13 +113,13 @@ TEST(PrimaryKeyTableUtilsTest, TestRealtimeRejectsDeleteAndSequenceOrderingOptio
 TEST(PrimaryKeyTableUtilsTest, TestRealtimeReportsSpecificLookupErrors) {
     const std::vector<std::pair<std::map<std::string, std::string>, std::string>> cases = {
         {{{Options::BUCKET, "1"}, {Options::FORCE_LOOKUP, "true"}},
-         "PK realtime v1 does not support lookup"},
+         "PK realtime does not support lookup"},
         {{{Options::BUCKET, "1"}, {Options::DELETION_VECTORS_ENABLED, "true"}},
-         "PK realtime v1 does not support deletion vectors"},
+         "PK realtime does not support deletion vectors"},
         {{{Options::BUCKET, "1"}, {Options::CHANGELOG_PRODUCER, "input"}},
-         "PK realtime v1 supports only the NONE changelog producer"},
+         "PK realtime supports only the NONE changelog producer"},
         {{{Options::BUCKET, "1"}, {Options::CHANGELOG_PRODUCER, "lookup"}},
-         "PK realtime v1 supports only the NONE changelog producer"},
+         "PK realtime supports only the NONE changelog producer"},
     };
     for (const auto& [option_map, expected_message] : cases) {
         ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));

From 3392f393b30360651f3c3ec1e55e843112312896 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Wed, 26 Aug 2026 22:30:20 +0800
Subject: [PATCH 46/62] refactor(realtime): simplify primary key schema and
 reader setup

---
 .../core/io/single_file_writer_test.cpp       |   4 +-
 .../operation/key_value_file_store_write.cpp  |  10 +-
 .../key_value_file_store_write_test.cpp       |  22 --
 .../realtime/prepared_key_value_reader.cpp    | 246 +++++++++++-------
 .../core/realtime/prepared_key_value_reader.h |   7 +
 .../realtime/primary_key_realtime_store.cpp   |   9 +-
 .../realtime/realtime_primary_key_writer.cpp  |  16 +-
 .../realtime/realtime_primary_key_writer.h    |   1 +
 .../table/source/key_value_table_read.cpp     |  64 ++---
 .../core/table/source/key_value_table_read.h  |   3 +
 10 files changed, 202 insertions(+), 180 deletions(-)

diff --git a/src/paimon/core/io/single_file_writer_test.cpp b/src/paimon/core/io/single_file_writer_test.cpp
index 4136702e8..78fce54c7 100644
--- a/src/paimon/core/io/single_file_writer_test.cpp
+++ b/src/paimon/core/io/single_file_writer_test.cpp
@@ -18,10 +18,8 @@
 
 #include "paimon/core/io/single_file_writer.h"
 
-#include <functional>
 #include <map>
-#include <memory>
-#include <string>
+#include <utility>
 
 #include "arrow/api.h"
 #include "arrow/c/abi.h"
diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp
index f431597a7..86f432998 100644
--- a/src/paimon/core/operation/key_value_file_store_write.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write.cpp
@@ -124,6 +124,7 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
     std::shared_ptr<CompactManager> compact_manager;
     std::shared_ptr<RealtimeContextImpl> realtime_context_impl;
     std::optional<RealtimeStoreState> realtime_store_state;
+    std::shared_ptr<arrow::Schema> prepared_schema;
     if (realtime_context_) {
         std::vector<std::pair<std::string, std::string>> partition_values;
         PAIMON_ASSIGN_OR_RAISE(partition_values,
@@ -131,12 +132,7 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
         partition_map =
             std::map<std::string, std::string>(partition_values.begin(), partition_values.end());
         PAIMON_ASSIGN_OR_RAISE(realtime_context_impl, RealtimeContextImpl::Cast(realtime_context_));
-        if (schema_->GetFieldByName(SpecialFields::RealtimeOffset().Name())) {
-            return Status::Invalid("PK real-time write schema contains reserved transport field " +
-                                   SpecialFields::RealtimeOffset().Name());
-        }
-        std::shared_ptr<arrow::Schema> prepared_schema =
-            SpecialFields::PreparedKeyValueSchema(schema_->fields());
+        prepared_schema = SpecialFields::PreparedKeyValueSchema(schema_->fields());
         auto c_write_schema = std::make_unique<ArrowSchema>();
         PAIMON_RETURN_NOT_OK_FROM_ARROW(
             arrow::ExportSchema(*prepared_schema, c_write_schema.get()));
@@ -169,7 +165,7 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
         return std::shared_ptr<BatchWriter>(std::move(writer));
     }
     return RealtimePrimaryKeyWriter::Create(
-        partition_map, bucket, schema_, trimmed_primary_keys, key_comparator_,
+        partition_map, bucket, schema_, prepared_schema, trimmed_primary_keys, key_comparator_,
         realtime_context_impl, realtime_store_state.value(), restore_max_seq_number, writer, pool_);
 }
 
diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp
index d52841f95..fc6bfff11 100644
--- a/src/paimon/core/operation/key_value_file_store_write_test.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp
@@ -479,28 +479,6 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeWrite) {
     ASSERT_OK(writer->Close());
 }
 
-TEST_F(KeyValueFileStoreWriteTest, TestRealtimeOffsetCollision) {
-    const std::map<std::string, std::string> options = {{Options::BUCKET, "1"},
-                                                        {Options::REALTIME_ENABLED, "true"}};
-    const std::shared_ptr<arrow::Schema> schema = arrow::schema({
-        arrow::field("id", arrow::int64(), false),
-        arrow::field("_REALTIME_OFFSET", arrow::int64()),
-    });
-    std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
-    ASSERT_TRUE(dir);
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<Catalog> catalog, Catalog::Create(dir->Str(), options));
-    ASSERT_OK(catalog->CreateDatabase("foo", {}, /*ignore_if_exists=*/false));
-    ArrowSchema c_schema;
-    ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok());
-    Status create_status =
-        catalog->CreateTable(Identifier("foo", "bar"), &c_schema,
-                             /*partition_keys=*/{}, /*primary_keys=*/{"id"}, options,
-                             /*ignore_if_exists=*/false);
-    ArrowSchemaRelease(&c_schema);
-    ASSERT_NOK_WITH_MSG(create_status,
-                        "field name '_REALTIME_OFFSET' in schema cannot be special field");
-}
-
 TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) {
     const std::map<std::string, std::string> options = {{Options::BUCKET, "1"},
                                                         {Options::REALTIME_ENABLED, "true"}};
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index 4a0a52920..40b54897a 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -21,6 +21,7 @@
 #include <cstdint>
 #include <memory>
 #include <optional>
+#include <unordered_map>
 #include <utility>
 #include <vector>
 
@@ -123,46 +124,29 @@ Status CheckPreparedField(const std::shared_ptr<arrow::Schema>& schema, int32_t
     return Status::OK();
 }
 
-Result<int32_t> FindFieldIndexByPaimonId(const arrow::FieldVector& fields, int32_t field_id) {
-    std::optional<int32_t> matching_index;
-    for (int32_t i = 0; i < static_cast<int32_t>(fields.size()); ++i) {
-        PAIMON_ASSIGN_OR_RAISE(int32_t candidate_id,
-                               NestedProjectionUtils::GetPaimonFieldId(fields[i]));
-        if (candidate_id == field_id) {
-            if (matching_index.has_value()) {
-                return Status::Invalid(
-                    fmt::format("duplicate field id {} in prepared schema", field_id));
-            }
-            matching_index = i;
-        }
-    }
-    if (matching_index.has_value()) {
-        return matching_index.value();
-    }
-    return Status::Invalid(fmt::format("cannot find field id {} in prepared schema", field_id));
-}
-
 Result<std::vector<int32_t>> ResolveFieldIndexes(
     const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::unordered_map<int32_t, int32_t>& field_indexes,
     const std::shared_ptr<arrow::Schema>& row_schema) {
-    arrow::FieldVector prepared_value_fields(
-        prepared_schema->fields().begin() + SpecialFields::kPreparedKeyValueValueStartIndex,
-        prepared_schema->fields().end());
     std::vector<int32_t> result;
     result.reserve(row_schema->num_fields());
     for (const std::shared_ptr<arrow::Field>& row_field : row_schema->fields()) {
         PAIMON_ASSIGN_OR_RAISE(int32_t field_id,
                                NestedProjectionUtils::GetPaimonFieldId(row_field));
-        PAIMON_ASSIGN_OR_RAISE(int32_t value_index,
-                               FindFieldIndexByPaimonId(prepared_value_fields, field_id));
-        const std::shared_ptr<arrow::Field>& prepared_field = prepared_value_fields[value_index];
+        auto field_index = field_indexes.find(field_id);
+        if (field_index == field_indexes.end()) {
+            return Status::Invalid(
+                fmt::format("cannot find field id {} in prepared schema", field_id));
+        }
+        const std::shared_ptr<arrow::Field>& prepared_field =
+            prepared_schema->field(field_index->second);
         if (!prepared_field->type()->Equals(row_field->type())) {
             return Status::Invalid(fmt::format(
                 "prepared field id {} type {} does not match row "
                 "type {}",
                 field_id, prepared_field->type()->ToString(), row_field->type()->ToString()));
         }
-        result.push_back(value_index + SpecialFields::kPreparedKeyValueValueStartIndex);
+        result.push_back(field_index->second);
     }
     return result;
 }
@@ -182,20 +166,81 @@ Status ValidateExactCommitSchema(const std::shared_ptr<arrow::Schema>& prepared_
     return Status::OK();
 }
 
+class PreparedReaderPlan {
+ public:
+    static Result<std::shared_ptr<const PreparedReaderPlan>> Create(
+        const std::shared_ptr<arrow::Schema>& prepared_schema,
+        const std::shared_ptr<arrow::Schema>& key_schema,
+        const std::shared_ptr<arrow::Schema>& value_schema,
+        const std::shared_ptr<MemoryPool>& memory_pool, bool exact_commit_schema) {
+        PAIMON_RETURN_NOT_OK(
+            PreparedKeyValueReaderFactory::ValidateTransportSchema(prepared_schema));
+        if (!key_schema) {
+            return Status::Invalid("prepared key schema cannot be null");
+        }
+        if (!value_schema) {
+            return Status::Invalid("prepared value schema cannot be null");
+        }
+        if (!memory_pool) {
+            return Status::Invalid("prepared reader memory pool cannot be null");
+        }
+        if (exact_commit_schema) {
+            PAIMON_RETURN_NOT_OK(ValidateExactCommitSchema(prepared_schema, value_schema));
+        }
+        std::unordered_map<int32_t, int32_t> field_indexes;
+        field_indexes.reserve(prepared_schema->num_fields() -
+                              SpecialFields::kPreparedKeyValueValueStartIndex);
+        for (int32_t i = SpecialFields::kPreparedKeyValueValueStartIndex;
+             i < prepared_schema->num_fields(); ++i) {
+            PAIMON_ASSIGN_OR_RAISE(int32_t field_id, NestedProjectionUtils::GetPaimonFieldId(
+                                                         prepared_schema->field(i)));
+            if (!field_indexes.emplace(field_id, i).second) {
+                return Status::Invalid(
+                    fmt::format("duplicate field id {} in prepared schema", field_id));
+            }
+        }
+        PAIMON_ASSIGN_OR_RAISE(std::vector<int32_t> key_field_indexes,
+                               ResolveFieldIndexes(prepared_schema, field_indexes, key_schema));
+        PAIMON_ASSIGN_OR_RAISE(std::vector<int32_t> value_field_indexes,
+                               ResolveFieldIndexes(prepared_schema, field_indexes, value_schema));
+        return std::shared_ptr<const PreparedReaderPlan>(new PreparedReaderPlan(
+            prepared_schema, std::move(key_field_indexes), std::move(value_field_indexes)));
+    }
+
+    const std::shared_ptr<arrow::Schema>& PreparedSchema() const {
+        return prepared_schema_;
+    }
+
+    const std::vector<int32_t>& KeyFieldIndexes() const {
+        return key_field_indexes_;
+    }
+
+    const std::vector<int32_t>& ValueFieldIndexes() const {
+        return value_field_indexes_;
+    }
+
+ private:
+    PreparedReaderPlan(const std::shared_ptr<arrow::Schema>& schema,
+                       std::vector<int32_t>&& key_indexes, std::vector<int32_t>&& value_indexes)
+        : prepared_schema_(schema),
+          key_field_indexes_(std::move(key_indexes)),
+          value_field_indexes_(std::move(value_indexes)) {}
+
+    const std::shared_ptr<arrow::Schema> prepared_schema_;
+    const std::vector<int32_t> key_field_indexes_;
+    const std::vector<int32_t> value_field_indexes_;
+};
+
 class PreparedKeyValueReader final : public KeyValueRecordReader {
  public:
     PreparedKeyValueReader(std::unique_ptr<BatchReader>&& reader,
-                           const std::shared_ptr<arrow::Schema>& prepared_schema,
+                           const std::shared_ptr<const PreparedReaderPlan>& plan,
                            const std::optional<OffsetRange>& visible_offsets,
-                           std::vector<int32_t>&& key_field_indexes,
-                           std::vector<int32_t>&& value_field_indexes,
                            const std::shared_ptr<MemoryPool>& pool,
                            const std::shared_ptr<RealtimeOffsetCoverage>& offset_coverage)
         : reader_(std::move(reader)),
-          prepared_schema_(prepared_schema),
+          plan_(plan),
           visible_offsets_(visible_offsets),
-          key_field_indexes_(std::move(key_field_indexes)),
-          value_field_indexes_(std::move(value_field_indexes)),
           pool_(pool),
           offset_coverage_(offset_coverage) {}
 
@@ -284,14 +329,6 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
             }
             std::shared_ptr<arrow::StructArray> data_batch =
                 checked_pointer_cast<arrow::StructArray>(arrow_array);
-            Status transport_status = PreparedKeyValueReaderFactory::ValidateTransportSchema(
-                arrow::schema(data_batch->type()->fields()));
-            if (!transport_status.ok()) {
-                return Status::Invalid(
-                    "prepared batch field does not match prepared transport "
-                    "schema: ",
-                    transport_status.ToString());
-            }
             PAIMON_RETURN_NOT_OK(ValidatePreparedBatch(data_batch));
 
             std::shared_ptr<arrow::NumericArray<arrow::Int64Type>> offset_array =
@@ -306,13 +343,13 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
             sequence_number_array_ = checked_pointer_cast<arrow::NumericArray<arrow::Int64Type>>(
                 data_batch->field(SpecialFields::kPreparedKeyValueSequenceNumberIndex));
             arrow::ArrayVector key_fields;
-            key_fields.reserve(key_field_indexes_.size());
-            for (int32_t index : key_field_indexes_) {
+            key_fields.reserve(plan_->KeyFieldIndexes().size());
+            for (int32_t index : plan_->KeyFieldIndexes()) {
                 key_fields.push_back(data_batch->field(index));
             }
             arrow::ArrayVector value_fields;
-            value_fields.reserve(value_field_indexes_.size());
-            for (int32_t index : value_field_indexes_) {
+            value_fields.reserve(plan_->ValueFieldIndexes().size());
+            for (int32_t index : plan_->ValueFieldIndexes()) {
                 value_fields.push_back(data_batch->field(index));
             }
             key_ctx_ = std::make_shared<ColumnarBatchContext>(key_fields, pool_);
@@ -328,33 +365,18 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
     }
 
     Status ValidatePreparedBatch(const std::shared_ptr<arrow::StructArray>& data_batch) const {
-        if (data_batch->num_fields() != prepared_schema_->num_fields()) {
+        if (data_batch->num_fields() != plan_->PreparedSchema()->num_fields()) {
             return Status::Invalid(fmt::format(
                 "prepared batch field count {} does not match prepared schema field count {}",
-                data_batch->num_fields(), prepared_schema_->num_fields()));
+                data_batch->num_fields(), plan_->PreparedSchema()->num_fields()));
         }
         const arrow::FieldVector& batch_fields = data_batch->type()->fields();
         for (int32_t i = 0; i < data_batch->num_fields(); ++i) {
-            if (!batch_fields[i]->Equals(prepared_schema_->field(i), true)) {
+            if (!batch_fields[i]->Equals(plan_->PreparedSchema()->field(i), true)) {
                 return Status::Invalid(fmt::format(
                     "prepared batch field {} does not match declared prepared schema", i));
             }
         }
-        if (!data_batch->field(SpecialFields::kPreparedKeyValueValueKindIndex) ||
-            data_batch->field(SpecialFields::kPreparedKeyValueValueKindIndex)->type_id() !=
-                arrow::Type::INT8) {
-            return Status::Invalid("cannot cast VALUE_KIND column to int8 arrow array");
-        }
-        if (!data_batch->field(SpecialFields::kPreparedKeyValueSequenceNumberIndex) ||
-            data_batch->field(SpecialFields::kPreparedKeyValueSequenceNumberIndex)->type_id() !=
-                arrow::Type::INT64) {
-            return Status::Invalid("cannot cast SEQUENCE_NUMBER column to int64 arrow array");
-        }
-        if (!data_batch->field(SpecialFields::kPreparedKeyValueRealtimeOffsetIndex) ||
-            data_batch->field(SpecialFields::kPreparedKeyValueRealtimeOffsetIndex)->type_id() !=
-                arrow::Type::INT64) {
-            return Status::Invalid("cannot cast REALTIME_OFFSET column to int64 arrow array");
-        }
         if (data_batch->field(SpecialFields::kPreparedKeyValueValueKindIndex)->null_count() != 0 ||
             data_batch->field(SpecialFields::kPreparedKeyValueSequenceNumberIndex)->null_count() !=
                 0 ||
@@ -411,10 +433,8 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
     bool closed_ = false;
     std::optional<Status> first_error_;
     std::unique_ptr<BatchReader> reader_;
-    std::shared_ptr<arrow::Schema> prepared_schema_;
+    std::shared_ptr<const PreparedReaderPlan> plan_;
     std::optional<OffsetRange> visible_offsets_;
-    std::vector<int32_t> key_field_indexes_;
-    std::vector<int32_t> value_field_indexes_;
     std::shared_ptr<MemoryPool> pool_;
     std::shared_ptr<RealtimeOffsetCoverage> offset_coverage_;
     bool offset_coverage_finished_ = false;
@@ -448,10 +468,8 @@ Status PreparedKeyValueReaderFactory::ValidateTransportSchema(
 namespace {
 
 Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderImpl(
-    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
+    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<const PreparedReaderPlan>& plan,
     const std::optional<OffsetRange>& visible_offsets,
-    const std::shared_ptr<arrow::Schema>& key_schema,
-    const std::shared_ptr<arrow::Schema>& value_schema,
     const std::shared_ptr<MemoryPool>& memory_pool,
     const std::shared_ptr<RealtimeOffsetCoverage>& offset_coverage) {
     std::unique_ptr<BatchReader> owned_reader = std::move(reader);
@@ -462,26 +480,8 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderImpl(
     if (visible_offsets.has_value() && visible_offsets->begin > visible_offsets->end) {
         return Status::Invalid("prepared visible offset range begin exceeds end");
     }
-    PAIMON_RETURN_NOT_OK(PreparedKeyValueReaderFactory::ValidateTransportSchema(prepared_schema));
-    if (!key_schema) {
-        return Status::Invalid("prepared key schema cannot be null");
-    }
-    if (!value_schema) {
-        return Status::Invalid("prepared value schema cannot be null");
-    }
-    if (!memory_pool) {
-        return Status::Invalid("prepared reader memory pool cannot be null");
-    }
-    if (!visible_offsets.has_value()) {
-        PAIMON_RETURN_NOT_OK(ValidateExactCommitSchema(prepared_schema, value_schema));
-    }
-    PAIMON_ASSIGN_OR_RAISE(std::vector<int32_t> key_field_indexes,
-                           ResolveFieldIndexes(prepared_schema, key_schema));
-    PAIMON_ASSIGN_OR_RAISE(std::vector<int32_t> value_field_indexes,
-                           ResolveFieldIndexes(prepared_schema, value_schema));
     std::unique_ptr<KeyValueRecordReader> result = std::make_unique<PreparedKeyValueReader>(
-        std::move(owned_reader), prepared_schema, visible_offsets, std::move(key_field_indexes),
-        std::move(value_field_indexes), memory_pool, offset_coverage);
+        std::move(owned_reader), plan, visible_offsets, memory_pool, offset_coverage);
     close_guard.Release();
     return result;
 }
@@ -494,9 +494,61 @@ Result<std::unique_ptr<KeyValueRecordReader>> PreparedKeyValueReaderFactory::Cre
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
     const std::shared_ptr<MemoryPool>& memory_pool) {
-    return AdaptPreparedBatchReaderImpl(std::move(reader), prepared_schema, visible_offsets,
-                                        key_schema, value_schema, memory_pool,
-                                        /*offset_coverage=*/nullptr);
+    std::unique_ptr<BatchReader> owned_reader = std::move(reader);
+    ScopeGuard reader_guard([&owned_reader]() {
+        if (owned_reader) {
+            owned_reader->Close();
+        }
+    });
+    if (visible_offsets.has_value() && visible_offsets->begin > visible_offsets->end) {
+        return Status::Invalid("prepared visible offset range begin exceeds end");
+    }
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<const PreparedReaderPlan> plan,
+        PreparedReaderPlan::Create(prepared_schema, key_schema, value_schema, memory_pool,
+                                   /*exact_commit_schema=*/!visible_offsets.has_value()));
+    PAIMON_ASSIGN_OR_RAISE(
+        std::unique_ptr<KeyValueRecordReader> result,
+        AdaptPreparedBatchReaderImpl(std::move(owned_reader), plan, visible_offsets, memory_pool,
+                                     /*offset_coverage=*/nullptr));
+    reader_guard.Release();
+    return result;
+}
+
+Result<std::vector<std::unique_ptr<KeyValueRecordReader>>>
+PreparedKeyValueReaderFactory::CreateForQuery(std::vector<std::unique_ptr<BatchReader>>&& readers,
+                                              const std::shared_ptr<arrow::Schema>& prepared_schema,
+                                              const OffsetRange& visible_offsets,
+                                              const std::shared_ptr<arrow::Schema>& key_schema,
+                                              const std::shared_ptr<arrow::Schema>& value_schema,
+                                              const std::shared_ptr<MemoryPool>& memory_pool) {
+    std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers;
+    ScopeGuard readers_guard([&readers, &adapted_readers]() {
+        CloseReaders(readers);
+        CloseReaders(adapted_readers);
+    });
+    if (visible_offsets.begin > visible_offsets.end) {
+        return Status::Invalid("prepared visible offset range begin exceeds end");
+    }
+    for (const std::unique_ptr<BatchReader>& reader : readers) {
+        if (!reader) {
+            return Status::Invalid("PK real-time store returned a null query reader");
+        }
+    }
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<const PreparedReaderPlan> plan,
+        PreparedReaderPlan::Create(prepared_schema, key_schema, value_schema, memory_pool,
+                                   /*exact_commit_schema=*/false));
+    adapted_readers.reserve(readers.size());
+    for (std::unique_ptr<BatchReader>& reader : readers) {
+        PAIMON_ASSIGN_OR_RAISE(
+            std::unique_ptr<KeyValueRecordReader> adapted_reader,
+            AdaptPreparedBatchReaderImpl(std::move(reader), plan, visible_offsets, memory_pool,
+                                         /*offset_coverage=*/nullptr));
+        adapted_readers.push_back(std::move(adapted_reader));
+    }
+    readers_guard.Release();
+    return adapted_readers;
 }
 
 Result<std::vector<std::unique_ptr<KeyValueRecordReader>>>
@@ -511,22 +563,22 @@ PreparedKeyValueReaderFactory::CreateForCommit(
         CloseReaders(readers);
         CloseReaders(adapted_readers);
     });
-    if (!memory_pool) {
-        return Status::Invalid("prepared reader memory pool cannot be null");
-    }
     for (const std::unique_ptr<BatchReader>& reader : readers) {
         if (!reader) {
             return Status::Invalid("PK real-time store returned a null commit reader");
         }
     }
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<const PreparedReaderPlan> plan,
+        PreparedReaderPlan::Create(prepared_schema, key_schema, value_schema, memory_pool,
+                                   /*exact_commit_schema=*/true));
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeOffsetCoverage> offset_coverage,
                            RealtimeOffsetCoverage::Create(sealed_offsets, readers.size()));
     adapted_readers.reserve(readers.size());
     for (std::unique_ptr<BatchReader>& reader : readers) {
-        PAIMON_ASSIGN_OR_RAISE(
-            std::unique_ptr<KeyValueRecordReader> adapted_reader,
-            AdaptPreparedBatchReaderImpl(std::move(reader), prepared_schema, std::nullopt,
-                                         key_schema, value_schema, memory_pool, offset_coverage));
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> adapted_reader,
+                               AdaptPreparedBatchReaderImpl(std::move(reader), plan, std::nullopt,
+                                                            memory_pool, offset_coverage));
         adapted_readers.push_back(std::move(adapted_reader));
     }
     readers_guard.Release();
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.h b/src/paimon/core/realtime/prepared_key_value_reader.h
index 389c75f9a..658ffec08 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.h
+++ b/src/paimon/core/realtime/prepared_key_value_reader.h
@@ -47,6 +47,13 @@ class PreparedKeyValueReaderFactory {
         const std::shared_ptr<arrow::Schema>& value_schema,
         const std::shared_ptr<MemoryPool>& memory_pool);
 
+    static Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateForQuery(
+        std::vector<std::unique_ptr<BatchReader>>&& readers,
+        const std::shared_ptr<arrow::Schema>& prepared_schema, const OffsetRange& visible_offsets,
+        const std::shared_ptr<arrow::Schema>& key_schema,
+        const std::shared_ptr<arrow::Schema>& value_schema,
+        const std::shared_ptr<MemoryPool>& memory_pool);
+
     static Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateForCommit(
         std::vector<std::unique_ptr<BatchReader>>&& readers,
         const std::shared_ptr<arrow::Schema>& prepared_schema, const OffsetRange& sealed_offsets,
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index dc3498424..a22ba8fed 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -138,11 +138,9 @@ class StoredBatchReader final : public BatchReader {
 
 class PrimaryKeyRealtimeStore::Impl {
  public:
-    Impl(std::shared_ptr<arrow::Schema> prepared_schema, std::shared_ptr<MemoryPool> memory_pool,
+    Impl(std::shared_ptr<arrow::Schema> prepared_schema,
          std::shared_ptr<arrow::MemoryPool> arrow_pool)
-        : prepared_schema_(std::move(prepared_schema)),
-          memory_pool_(std::move(memory_pool)),
-          arrow_pool_(std::move(arrow_pool)) {}
+        : prepared_schema_(std::move(prepared_schema)), arrow_pool_(std::move(arrow_pool)) {}
 
     Status Write(RealtimeWriteBatch&& write_batch) {
         if (!write_batch.batch || !write_batch.batch->GetData()) {
@@ -263,7 +261,6 @@ class PrimaryKeyRealtimeStore::Impl {
 
  private:
     std::shared_ptr<arrow::Schema> prepared_schema_;
-    std::shared_ptr<MemoryPool> memory_pool_;
     std::shared_ptr<arrow::MemoryPool> arrow_pool_;
     mutable std::mutex mutex_;
     std::vector<StoredBatch> building_;
@@ -284,7 +281,7 @@ Result<std::shared_ptr<PrimaryKeyRealtimeStore>> PrimaryKeyRealtimeStore::Create
     }
     std::shared_ptr<arrow::MemoryPool> arrow_pool = GetArrowPool(memory_pool);
     return std::shared_ptr<PrimaryKeyRealtimeStore>(new PrimaryKeyRealtimeStore(
-        std::make_unique<Impl>(prepared_schema, memory_pool, std::move(arrow_pool))));
+        std::make_unique<Impl>(prepared_schema, std::move(arrow_pool))));
 }
 Status PrimaryKeyRealtimeStore::Write(RealtimeWriteBatch&& batch) {
     return impl_->Write(std::move(batch));
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index ae123a643..58103c599 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -100,11 +100,7 @@ Result<std::shared_ptr<arrow::StructArray>> PrepareBatch(
         arrow::Datum sorted,
         arrow::compute::Take(arrow::Datum(prepared), indices,
                              arrow::compute::TakeOptions::NoBoundsCheck(), &context));
-    std::shared_ptr<arrow::Array> sorted_array = sorted.make_array();
-    if (!sorted_array || sorted_array->type_id() != arrow::Type::STRUCT) {
-        return Status::Invalid("PK real-time sorted batch is not a StructArray");
-    }
-    return checked_pointer_cast<arrow::StructArray>(std::move(sorted_array));
+    return checked_pointer_cast<arrow::StructArray>(sorted.make_array());
 }
 
 }  // namespace
@@ -112,19 +108,13 @@ Result<std::shared_ptr<arrow::StructArray>> PrepareBatch(
 Result<std::shared_ptr<RealtimePrimaryKeyWriter>> RealtimePrimaryKeyWriter::Create(
     const std::map<std::string, std::string>& partition, int32_t bucket,
     const std::shared_ptr<arrow::Schema>& write_schema,
+    const std::shared_ptr<arrow::Schema>& prepared_schema,
     const std::vector<std::string>& trimmed_primary_keys,
     const std::shared_ptr<FieldsComparator>& key_comparator,
     const std::shared_ptr<RealtimeContextImpl>& realtime_context,
     const RealtimeStoreState& store_state, int64_t restored_max_sequence_number,
     const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
     const std::shared_ptr<MemoryPool>& memory_pool) {
-    if (!store_state.store || !merge_tree_writer || !write_schema || !key_comparator ||
-        !realtime_context || !memory_pool) {
-        return Status::Invalid("PK real-time writer received a null dependency");
-    }
-    if (trimmed_primary_keys.empty()) {
-        return Status::Invalid("PK real-time writer requires at least one primary key");
-    }
     if (restored_max_sequence_number < -1 ||
         restored_max_sequence_number == std::numeric_limits<int64_t>::max()) {
         return Status::Invalid("PK restored sequence number is invalid");
@@ -138,8 +128,6 @@ Result<std::shared_ptr<RealtimePrimaryKeyWriter>> RealtimePrimaryKeyWriter::Crea
         }
         key_fields.push_back(std::move(field));
     }
-    std::shared_ptr<arrow::Schema> prepared_schema =
-        SpecialFields::PreparedKeyValueSchema(write_schema->fields());
     const RealtimePartitionBucket partition_bucket(partition, bucket);
     PAIMON_ASSIGN_OR_RAISE(int64_t initial_max_sequence_number,
                            realtime_context->AdvanceMaterializedMaxSequenceNumber(
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.h b/src/paimon/core/realtime/realtime_primary_key_writer.h
index 5cae9ca47..4a26f930d 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.h
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.h
@@ -48,6 +48,7 @@ class RealtimePrimaryKeyWriter final : public BatchWriter {
     static Result<std::shared_ptr<RealtimePrimaryKeyWriter>> Create(
         const std::map<std::string, std::string>& partition, int32_t bucket,
         const std::shared_ptr<arrow::Schema>& write_schema,
+        const std::shared_ptr<arrow::Schema>& prepared_schema,
         const std::vector<std::string>& trimmed_primary_keys,
         const std::shared_ptr<FieldsComparator>& key_comparator,
         const std::shared_ptr<RealtimeContextImpl>& realtime_context,
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index e4225587d..110994331 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -56,13 +56,9 @@ struct ColumnarBatchContext;
 
 namespace {
 
-Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateMemoryReaders(
-    const std::shared_ptr<RealtimeSplit>& split, const RealtimePartitionBucketView& memory,
+Result<std::shared_ptr<arrow::Schema>> CreatePreparedQuerySchema(
     const std::shared_ptr<arrow::Schema>& key_schema,
-    const std::shared_ptr<arrow::Schema>& value_schema,
-    const std::shared_ptr<FieldsComparator>& key_comparator,
-    const std::shared_ptr<InternalReadContext>& context,
-    const std::shared_ptr<MemoryPool>& memory_pool) {
+    const std::shared_ptr<arrow::Schema>& value_schema) {
     arrow::FieldVector prepared_value_fields;
     prepared_value_fields.reserve(key_schema->num_fields() + value_schema->num_fields());
     std::unordered_set<int32_t> field_ids;
@@ -78,33 +74,31 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateMemoryReaders(
             prepared_value_fields.push_back(field);
         }
     }
-    std::shared_ptr<arrow::Schema> prepared_schema =
-        SpecialFields::PreparedKeyValueSchema(prepared_value_fields);
+    return SpecialFields::PreparedKeyValueSchema(prepared_value_fields);
+}
+
+Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateMemoryReaders(
+    const std::shared_ptr<RealtimeSplit>& split, const RealtimePartitionBucketView& memory,
+    const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::shared_ptr<arrow::Schema>& key_schema,
+    const std::shared_ptr<arrow::Schema>& value_schema,
+    const std::shared_ptr<FieldsComparator>& key_comparator,
+    const std::shared_ptr<InternalReadContext>& context,
+    const std::shared_ptr<MemoryPool>& memory_pool) {
     auto c_schema = std::make_unique<ArrowSchema>();
     PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*prepared_schema, c_schema.get()));
     ScopeGuard schema_guard([schema = c_schema.get()]() { ArrowSchemaRelease(schema); });
     RealtimeQueryContext query_context{c_schema.get(), nullptr, false};
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> batch_readers,
                            memory.store->CreateQueryReaders(memory.read_view, 0, query_context));
-    ScopeGuard batch_readers_guard([&batch_readers]() {
-        for (const std::unique_ptr<BatchReader>& reader : batch_readers) {
-            if (reader) {
-                reader->Close();
-            }
-        }
-    });
+    PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> prepared_readers,
+                           PreparedKeyValueReaderFactory::CreateForQuery(
+                               std::move(batch_readers), prepared_schema,
+                               OffsetRange(split->CommittedEndOffset(), split->MemoryEndOffset()),
+                               key_schema, value_schema, memory_pool));
     std::vector<std::unique_ptr<KeyValueRecordReader>> result;
-    result.reserve(batch_readers.size());
-    for (std::unique_ptr<BatchReader>& reader : batch_readers) {
-        if (!reader) {
-            return Status::Invalid("PK real-time store returned a null query reader");
-        }
-        PAIMON_ASSIGN_OR_RAISE(
-            std::unique_ptr<KeyValueRecordReader> prepared_reader,
-            PreparedKeyValueReaderFactory::Create(
-                std::move(reader), prepared_schema,
-                OffsetRange(split->CommittedEndOffset(), split->MemoryEndOffset()), key_schema,
-                value_schema, memory_pool));
+    result.reserve(prepared_readers.size());
+    for (std::unique_ptr<KeyValueRecordReader>& prepared_reader : prepared_readers) {
         PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<MergeFunction> merge,
                                PrimaryKeyTableUtils::CreateMergeFunction(
                                    value_schema, context->GetTableSchema()->PrimaryKeys(),
@@ -113,7 +107,6 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateMemoryReaders(
             std::move(prepared_reader), key_comparator,
             std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge))));
     }
-    batch_readers_guard.Release();
     return result;
 }
 
@@ -122,12 +115,14 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateMemoryReaders(
 KeyValueTableRead::KeyValueTableRead(std::vector<std::unique_ptr<SplitRead>>&& split_reads,
                                      const std::shared_ptr<FileStorePathFactory>& path_factory,
                                      const std::shared_ptr<InternalReadContext>& context,
+                                     const std::shared_ptr<arrow::Schema>& prepared_query_schema,
                                      const std::shared_ptr<MemoryPool>& memory_pool,
                                      const std::shared_ptr<Executor>& executor)
     : TableRead(memory_pool),
       split_reads_(std::move(split_reads)),
       path_factory_(path_factory),
       context_(context),
+      prepared_query_schema_(prepared_query_schema),
       executor_(executor) {}
 
 Result<std::unique_ptr<TableRead>> KeyValueTableRead::Create(
@@ -141,10 +136,17 @@ Result<std::unique_ptr<TableRead>> KeyValueTableRead::Create(
     PAIMON_ASSIGN_OR_RAISE(
         std::unique_ptr<MergeFileSplitRead> merge_file_split_read,
         MergeFileSplitRead::Create(path_factory, context, memory_pool, executor));
+    std::shared_ptr<arrow::Schema> prepared_query_schema;
+    if (context->GetRealtimeContext()) {
+        PAIMON_ASSIGN_OR_RAISE(prepared_query_schema,
+                               CreatePreparedQuerySchema(merge_file_split_read->GetKeySchema(),
+                                                         merge_file_split_read->GetValueSchema()));
+    }
     split_reads.emplace_back(std::move(merge_file_split_read));
 
     return std::unique_ptr<TableRead>(new KeyValueTableRead(std::move(split_reads), path_factory,
-                                                            context, memory_pool, executor));
+                                                            context, prepared_query_schema,
+                                                            memory_pool, executor));
 }
 
 void KeyValueTableRead::ForceKeepDelete(bool force_keep_delete) {
@@ -287,9 +289,9 @@ Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateRealtimeReader(
         if (merge_read) {
             PAIMON_ASSIGN_OR_RAISE(
                 std::vector<std::unique_ptr<KeyValueRecordReader>> memory_readers,
-                CreateMemoryReaders(realtime_split, memory, merge_read->GetKeySchema(),
-                                    merge_read->GetValueSchema(), merge_read->GetKeyComparator(),
-                                    context_, GetMemoryPool()));
+                CreateMemoryReaders(realtime_split, memory, prepared_query_schema_,
+                                    merge_read->GetKeySchema(), merge_read->GetValueSchema(),
+                                    merge_read->GetKeyComparator(), context_, GetMemoryPool()));
             PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> reader,
                                    merge_read->CreateRealtimeReader(realtime_split->DiskSplits(),
                                                                     std::move(memory_readers)));
diff --git a/src/paimon/core/table/source/key_value_table_read.h b/src/paimon/core/table/source/key_value_table_read.h
index 6824ae59e..54f802cf6 100644
--- a/src/paimon/core/table/source/key_value_table_read.h
+++ b/src/paimon/core/table/source/key_value_table_read.h
@@ -22,6 +22,7 @@
 #include <memory>
 #include <vector>
 
+#include "arrow/type_fwd.h"
 #include "paimon/core/operation/internal_read_context.h"
 #include "paimon/core/operation/split_read.h"
 #include "paimon/core/utils/file_store_path_factory.h"
@@ -58,6 +59,7 @@ class KeyValueTableRead : public TableRead {
     KeyValueTableRead(std::vector<std::unique_ptr<SplitRead>>&& split_reads,
                       const std::shared_ptr<FileStorePathFactory>& path_factory,
                       const std::shared_ptr<InternalReadContext>& context,
+                      const std::shared_ptr<arrow::Schema>& prepared_query_schema,
                       const std::shared_ptr<MemoryPool>& memory_pool,
                       const std::shared_ptr<Executor>& executor);
 
@@ -67,6 +69,7 @@ class KeyValueTableRead : public TableRead {
     std::vector<std::unique_ptr<SplitRead>> split_reads_;
     std::shared_ptr<FileStorePathFactory> path_factory_;
     std::shared_ptr<InternalReadContext> context_;
+    std::shared_ptr<arrow::Schema> prepared_query_schema_;
     std::shared_ptr<Executor> executor_;
     bool force_keep_delete_ = false;
 };

From dfb332948580ffb35838060ae269274c8600e76f Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Wed, 26 Aug 2026 22:54:32 +0800
Subject: [PATCH 47/62] refactor(realtime): remove duplicate offset validation

---
 src/paimon/core/realtime/prepared_key_value_reader.cpp | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index 40b54897a..8620ee872 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -477,9 +477,6 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderImpl(
         return Status::Invalid("prepared batch reader cannot be null");
     }
     ScopeGuard close_guard([&owned_reader]() -> void { owned_reader->Close(); });
-    if (visible_offsets.has_value() && visible_offsets->begin > visible_offsets->end) {
-        return Status::Invalid("prepared visible offset range begin exceeds end");
-    }
     std::unique_ptr<KeyValueRecordReader> result = std::make_unique<PreparedKeyValueReader>(
         std::move(owned_reader), plan, visible_offsets, memory_pool, offset_coverage);
     close_guard.Release();

From e448f7ac5324351479fba88021aacdba040d19fb Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Thu, 27 Aug 2026 15:04:16 +0800
Subject: [PATCH 48/62] refactor(realtime): localize primary-key split
 validation

---
 .../core/operation/merge_file_split_read.cpp  | 55 ++++++++++---------
 .../operation/merge_file_split_read_test.cpp  | 54 ++++++++++++++++++
 2 files changed, 82 insertions(+), 27 deletions(-)

diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp
index 6c8bbecad..3b09d8c54 100644
--- a/src/paimon/core/operation/merge_file_split_read.cpp
+++ b/src/paimon/core/operation/merge_file_split_read.cpp
@@ -149,28 +149,28 @@ class MergeFileSplitRead::RealtimeReaderBuilder {
 
     Status CollectDiskReaders(const std::vector<std::shared_ptr<Split>>& disk_splits,
                               std::vector<std::unique_ptr<KeyValueRecordReader>>* readers) {
-        std::shared_ptr<DataSplitImpl> first_split =
-            std::dynamic_pointer_cast<DataSplitImpl>(disk_splits.front());
-        if (!first_split) {
-            return Status::Invalid("merge input disk split is not a data split");
+        std::vector<std::shared_ptr<DataSplitImpl>> data_splits;
+        data_splits.reserve(disk_splits.size());
+        for (const std::shared_ptr<Split>& disk_split : disk_splits) {
+            std::shared_ptr<DataSplitImpl> data_split =
+                std::dynamic_pointer_cast<DataSplitImpl>(disk_split);
+            if (!data_split) {
+                return Status::Invalid("merge input disk split is not a data split");
+            }
+            data_splits.push_back(std::move(data_split));
         }
+        const std::shared_ptr<DataSplitImpl>& first_split = data_splits.front();
         const BinaryRow& partition = first_split->Partition();
         const int32_t bucket = first_split->Bucket();
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<DataFilePathFactory> data_file_path_factory,
-                               owner_->path_factory_->CreateDataFilePathFactory(partition, bucket));
 
         std::vector<std::shared_ptr<DataFileMeta>> data_files;
         std::vector<std::optional<DeletionFile>> deletion_files;
-        for (const std::shared_ptr<Split>& disk_split : disk_splits) {
-            std::shared_ptr<DataSplitImpl> data_split =
-                std::dynamic_pointer_cast<DataSplitImpl>(disk_split);
-            if (!data_split || !(data_split->Partition() == partition) ||
-                data_split->Bucket() != bucket) {
+        for (const std::shared_ptr<DataSplitImpl>& data_split : data_splits) {
+            if (!(data_split->Partition() == partition) || data_split->Bucket() != bucket) {
                 return Status::Invalid("merge input disk splits do not share a partition-bucket");
             }
-            if (!data_split->BeforeFiles().empty() || data_split->IsStreaming() ||
-                data_split->Bucket() == BucketModeDefine::POSTPONE_BUCKET) {
-                return Status::Invalid("additional merge input requires fixed-bucket batch splits");
+            if (!data_split->BeforeFiles().empty()) {
+                return Status::Invalid("merge input disk split must not contain before files");
             }
             const std::vector<std::shared_ptr<DataFileMeta>>& split_files = data_split->DataFiles();
             const std::vector<std::optional<DeletionFile>>& split_deletion_files =
@@ -188,11 +188,16 @@ class MergeFileSplitRead::RealtimeReaderBuilder {
                                       split_deletion_files.end());
             }
         }
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<DataFilePathFactory> data_file_path_factory,
+                               owner_->path_factory_->CreateDataFilePathFactory(partition, bucket));
 
         DeletionVector::Factory dv_factory;
         std::vector<std::vector<SortedRun>> disk_sections;
         PAIMON_RETURN_NOT_OK(
             owner_->CreateDiskSections(data_files, deletion_files, &dv_factory, &disk_sections));
+        if (disk_sections.empty()) {
+            return Status::OK();
+        }
         std::vector<std::unique_ptr<KeyValueRecordReader>> section_readers;
         ScopeGuard section_readers_guard([&section_readers]() {
             for (const std::unique_ptr<KeyValueRecordReader>& reader : section_readers) {
@@ -200,13 +205,11 @@ class MergeFileSplitRead::RealtimeReaderBuilder {
             }
         });
         section_readers.reserve(disk_sections.size());
-        std::shared_ptr<MergeFunctionWrapper<KeyValue>> merge_function_wrapper;
-        if (!disk_sections.empty()) {
-            PAIMON_ASSIGN_OR_RAISE(merge_function_wrapper,
-                                   MergeFileSplitRead::CreateMergeFunctionWrapper(
-                                       owner_->options_, owner_->context_->GetTableSchema(),
-                                       owner_->value_schema_, owner_->pool_));
-        }
+        PAIMON_ASSIGN_OR_RAISE(
+            std::shared_ptr<MergeFunctionWrapper<KeyValue>> merge_function_wrapper,
+            MergeFileSplitRead::CreateMergeFunctionWrapper(owner_->options_,
+                                                           owner_->context_->GetTableSchema(),
+                                                           owner_->value_schema_, owner_->pool_));
         for (const std::vector<SortedRun>& section : disk_sections) {
             PAIMON_ASSIGN_OR_RAISE(
                 std::unique_ptr<SortMergeReader> section_reader,
@@ -216,12 +219,10 @@ class MergeFileSplitRead::RealtimeReaderBuilder {
             section_readers.push_back(
                 std::make_unique<SortMergeKeyValueRecordReader>(std::move(section_reader)));
         }
-        if (!section_readers.empty()) {
-            std::unique_ptr<KeyValueRecordReader> concat_reader =
-                std::make_unique<ConcatKeyValueRecordReader>(std::move(section_readers));
-            section_readers_guard.Release();
-            readers->push_back(std::move(concat_reader));
-        }
+        std::unique_ptr<KeyValueRecordReader> concat_reader =
+            std::make_unique<ConcatKeyValueRecordReader>(std::move(section_readers));
+        section_readers_guard.Release();
+        readers->push_back(std::move(concat_reader));
         return Status::OK();
     }
 
diff --git a/src/paimon/core/operation/merge_file_split_read_test.cpp b/src/paimon/core/operation/merge_file_split_read_test.cpp
index a899fd904..d1da3e4f6 100644
--- a/src/paimon/core/operation/merge_file_split_read_test.cpp
+++ b/src/paimon/core/operation/merge_file_split_read_test.cpp
@@ -67,6 +67,12 @@ class FileSystem;
 }  // namespace paimon
 
 namespace paimon::test {
+namespace {
+
+class TestingSplit : public Split {};
+
+}  // namespace
+
 // Parameter: min_heap/loser_tree; enable/disable IO prefetch; enable/disable multi thread row to
 // batch
 class MergeFileSplitReadTest : public ::testing::Test,
@@ -739,6 +745,54 @@ TEST_P(MergeFileSplitReadTest, TestRealtimeReadConcatenatesOrderedDiskSections)
     batch_reader->Close();
 }
 
+TEST_P(MergeFileSplitReadTest, TestRealtimeReadValidatesDiskSplits) {
+    std::string path =
+        paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor";
+    ReadContextBuilder context_builder(path);
+    context_builder.SetReadFieldNames({"k0", "k1", "s1", "v0"});
+    context_builder.SetOptions(
+        {{Options::SEQUENCE_FIELD, "s0,s1"}, {Options::MERGE_ENGINE, "deduplicate"}});
+    AddOptions(&context_builder);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<ReadContext> read_context, context_builder.Finish());
+    std::shared_ptr<InternalReadContext> internal_context = CreateInternalReadContext(read_context);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<MergeFileSplitRead> split_read,
+                         CreateMergeFileSplitRead(internal_context));
+
+    std::vector<std::shared_ptr<DataSplit>> prepared_splits = PrepareDataSplit();
+    std::shared_ptr<DataSplitImpl> first =
+        std::dynamic_pointer_cast<DataSplitImpl>(prepared_splits[0]);
+    ASSERT_NE(nullptr, first);
+
+    std::vector<std::shared_ptr<Split>> non_data_splits = {std::make_shared<TestingSplit>()};
+    ASSERT_NOK_WITH_MSG(split_read->CreateRealtimeReader(non_data_splits, {}),
+                        "merge input disk split is not a data split");
+    std::vector<std::shared_ptr<Split>> mixed_partition_splits = {prepared_splits[0],
+                                                                 prepared_splits[1]};
+    ASSERT_NOK_WITH_MSG(split_read->CreateRealtimeReader(mixed_partition_splits, {}),
+                        "merge input disk splits do not share a partition-bucket");
+
+    std::vector<std::shared_ptr<DataFileMeta>> before_data_files = first->DataFiles();
+    DataSplitImpl::Builder before_builder(first->Partition(), first->Bucket(), first->BucketPath(),
+                                          std::move(before_data_files));
+    std::vector<std::shared_ptr<DataFileMeta>> before_files = first->DataFiles();
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr<DataSplitImpl> before_split,
+        before_builder.WithBeforeFiles(std::move(before_files)).RawConvertible(false).Build());
+    std::vector<std::shared_ptr<Split>> before_splits = {before_split};
+    ASSERT_NOK_WITH_MSG(split_read->CreateRealtimeReader(before_splits, {}),
+                        "merge input disk split must not contain before files");
+
+    std::vector<std::shared_ptr<DataFileMeta>> deletion_data_files = first->DataFiles();
+    DataSplitImpl::Builder deletion_builder(first->Partition(), first->Bucket(),
+                                            first->BucketPath(), std::move(deletion_data_files));
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr<DataSplitImpl> deletion_split,
+        deletion_builder.WithDataDeletionFiles({std::nullopt}).RawConvertible(false).Build());
+    std::vector<std::shared_ptr<Split>> deletion_splits = {deletion_split};
+    ASSERT_NOK_WITH_MSG(split_read->CreateRealtimeReader(deletion_splits, {}),
+                        "deletion files must be empty or match data files");
+}
+
 TEST_P(MergeFileSplitReadTest, TestLookUp) {
     std::string path =
         paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor";

From 0082544d86962f45e61216b84d945a7db403a46a Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Thu, 27 Aug 2026 16:38:18 +0800
Subject: [PATCH 49/62] fix(io): preserve merged reader initialization errors

---
 .../io/merged_key_value_record_reader.cpp     |  2 +-
 .../merged_key_value_record_reader_test.cpp   | 77 +++++++++++++++----
 2 files changed, 62 insertions(+), 17 deletions(-)

diff --git a/src/paimon/core/io/merged_key_value_record_reader.cpp b/src/paimon/core/io/merged_key_value_record_reader.cpp
index 8c3952874..8730086f5 100644
--- a/src/paimon/core/io/merged_key_value_record_reader.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader.cpp
@@ -123,7 +123,6 @@ Result<std::unique_ptr<KeyValueRecordReader::Iterator>> MergedKeyValueRecordRead
     if (visited_) {
         return std::unique_ptr<KeyValueRecordReader::Iterator>();
     }
-    visited_ = true;
 
     auto iterator = std::make_unique<Iterator>(this);
     Result<bool> has_next_result = iterator->HasNext();
@@ -132,6 +131,7 @@ Result<std::unique_ptr<KeyValueRecordReader::Iterator>> MergedKeyValueRecordRead
         return initialization_error_.value();
     }
     bool has_next = std::move(has_next_result).value();
+    visited_ = true;
     if (!has_next) {
         return std::unique_ptr<KeyValueRecordReader::Iterator>();
     }
diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index 1a94aee0e..b35e0c111 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -128,6 +128,54 @@ class MalformedBitmapBatchReader : public BatchReader {
     int32_t row_id_;
 };
 
+class ScriptedKeyValueRecordReader final : public KeyValueRecordReader {
+ public:
+    ScriptedKeyValueRecordReader(std::vector<KeyValue>&& key_values, int32_t* next_batch_count)
+        : key_values_(std::move(key_values)), next_batch_count_(next_batch_count) {}
+
+    class Iterator final : public KeyValueRecordReader::Iterator {
+     public:
+        explicit Iterator(KeyValue&& key_value) : key_value_(std::move(key_value)) {}
+
+        Result<bool> HasNext() const override {
+            return key_value_.has_value();
+        }
+
+        Result<KeyValue> Next() override {
+            KeyValue result = std::move(key_value_.value());
+            key_value_.reset();
+            return result;
+        }
+
+     private:
+        std::optional<KeyValue> key_value_;
+    };
+
+    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatch() override {
+        ++(*next_batch_count_);
+        if (*next_batch_count_ == 1) {
+            return std::make_unique<Iterator>(std::move(key_values_[0]));
+        }
+        if (*next_batch_count_ == 2) {
+            return Status::IOError("scripted lookahead failure");
+        }
+        if (*next_batch_count_ == 3) {
+            return std::make_unique<Iterator>(std::move(key_values_[1]));
+        }
+        return std::unique_ptr<KeyValueRecordReader::Iterator>();
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return nullptr;
+    }
+
+    void Close() override {}
+
+ private:
+    std::vector<KeyValue> key_values_;
+    int32_t* next_batch_count_;
+};
+
 }  // namespace
 
 class MergedKeyValueRecordReaderTest : public testing::Test {
@@ -459,29 +507,26 @@ TEST_F(MergedKeyValueRecordReaderTest, TestQueryReaderRequiresStoreAlignedSchema
     ASSERT_NOK_WITH_MSG(reader->NextBatch(), "field count");
 }
 
-TEST_F(MergedKeyValueRecordReaderTest, TestMergedReaderErrorRetry) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1]])").ValueOrDie();
-    auto failing_reader = std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
-    failing_reader->SetNextBatchStatus(Status::IOError("stable prepared error"));
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReaderForTest(std::move(failing_reader), prepared_schema,
-                                        OffsetRange(0, 1), value_schema, value_schema, pool_));
+TEST_F(MergedKeyValueRecordReaderTest, TestMergedReaderInitializationErrorIsTerminal) {
+    std::vector<DataField> key_fields = {DataField(0, arrow::field("key", arrow::int32()))};
+    std::vector<KeyValue> key_values =
+        KeyValueChecker::GenerateKeyValues({10, 11}, {{1}, {2}}, {{1}, {2}}, pool_);
+    int32_t next_batch_count = 0;
+    auto reader =
+        std::make_unique<ScriptedKeyValueRecordReader>(std::move(key_values), &next_batch_count);
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<FieldsComparator> key_comparator,
-                         FieldsComparator::Create({DataField(0, key)}, true));
+                         FieldsComparator::Create(key_fields, true));
     MergedKeyValueRecordReader merged_reader(std::move(reader), key_comparator,
                                              merge_function_wrapper_);
 
     Result<std::unique_ptr<KeyValueRecordReader::Iterator>> first = merged_reader.NextBatch();
     Result<std::unique_ptr<KeyValueRecordReader::Iterator>> retry = merged_reader.NextBatch();
-    ASSERT_NOK(first);
-    ASSERT_NOK(retry);
+    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> second_retry =
+        merged_reader.NextBatch();
+    ASSERT_NOK_WITH_MSG(first, "scripted lookahead failure");
     ASSERT_EQ(first.status().ToString(), retry.status().ToString());
+    ASSERT_EQ(first.status().ToString(), second_retry.status().ToString());
+    ASSERT_EQ(2, next_batch_count);
 }
 
 TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderSafeDecode) {

From b4fb4d07d6d2cdd4844735bf076a85e95c6903a7 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Thu, 27 Aug 2026 16:38:41 +0800
Subject: [PATCH 50/62] fix(realtime): validate prepared reader coverage

---
 .../merged_key_value_record_reader_test.cpp   | 316 +++++++++++++++---
 .../realtime/prepared_key_value_reader.cpp    | 184 +++++-----
 .../core/realtime/prepared_key_value_reader.h |  10 -
 test/inte/realtime_write_inte_test.cpp        |  68 +++-
 4 files changed, 403 insertions(+), 175 deletions(-)

diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index b35e0c111..8cdef7a6b 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -65,14 +65,32 @@ std::shared_ptr<arrow::Schema> MakePreparedSchema(const arrow::FieldVector& valu
     return arrow::schema(prepared_fields);
 }
 
-Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderForTest(
+Result<std::unique_ptr<KeyValueRecordReader>> CreatePreparedQueryReaderForTest(
     std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
-    const std::optional<OffsetRange>& visible_offsets,
-    const std::shared_ptr<arrow::Schema>& key_schema,
+    const OffsetRange& visible_offsets, const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
     const std::shared_ptr<MemoryPool>& memory_pool) {
-    return PreparedKeyValueReaderFactory::Create(
-        std::move(reader), prepared_schema, visible_offsets, key_schema, value_schema, memory_pool);
+    std::vector<std::unique_ptr<BatchReader>> readers;
+    readers.push_back(std::move(reader));
+    PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers,
+                           PreparedKeyValueReaderFactory::CreateForQuery(
+                               std::move(readers), prepared_schema, visible_offsets, key_schema,
+                               value_schema, memory_pool));
+    return std::move(adapted_readers[0]);
+}
+
+Result<std::unique_ptr<KeyValueRecordReader>> CreatePreparedCommitReaderForTest(
+    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const OffsetRange& sealed_offsets, const std::shared_ptr<arrow::Schema>& key_schema,
+    const std::shared_ptr<arrow::Schema>& value_schema,
+    const std::shared_ptr<MemoryPool>& memory_pool) {
+    std::vector<std::unique_ptr<BatchReader>> readers;
+    readers.push_back(std::move(reader));
+    PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers,
+                           PreparedKeyValueReaderFactory::CreateForCommit(
+                               std::move(readers), prepared_schema, sealed_offsets, key_schema,
+                               value_schema, memory_pool));
+    return std::move(adapted_readers[0]);
 }
 
 class TrackingBatchReader : public BatchReader {
@@ -283,7 +301,7 @@ TEST_F(MergedKeyValueRecordReaderTest, TestSkipMergedNulloptResultInHasNext) {
     }
 }
 
-TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderOffsetFilter) {
+TEST_F(MergedKeyValueRecordReaderTest, TestQueryAllowsCommittedPrefix) {
     std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32())),
                                            DataField(1, arrow::field("v0", arrow::int32()))};
     std::shared_ptr<arrow::Schema> value_schema =
@@ -295,31 +313,154 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderOffsetFilter) {
         arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
         [0, 100, 0, 1, 10],
         [0, 101, 1, 2, 20],
-        [0, 102, 4, 3, 30],
-        [0, 103, 2, 4, 40],
-        [0, 104, 5, 5, 50],
-        [0, 105, 3, 6, 60]
+        [0, 102, 2, 4, 40],
+        [0, 103, 3, 6, 60]
     ])")
             .ValueOrDie());
 
-    auto batch_reader = std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 2);
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, OffsetRange(2, 4),
-                                        key_schema, value_schema, pool_));
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+    batch_readers.push_back(
+        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 2));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
+                         PreparedKeyValueReaderFactory::CreateForQuery(
+                             std::move(batch_readers), prepared_schema, OffsetRange(2, 4),
+                             key_schema, value_schema, pool_));
+    ASSERT_EQ(1, readers.size());
     ASSERT_OK_AND_ASSIGN(
         std::vector<KeyValue> results,
-        (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
-                                                    KeyValueRecordReader::Iterator>(reader.get())));
+        (ReadResultCollector::CollectKeyValueResult<
+            KeyValueRecordReader, KeyValueRecordReader::Iterator>(readers[0].get())));
 
     std::vector<RowKind*> row_kinds = {const_cast<RowKind*>(RowKind::Insert()),
                                        const_cast<RowKind*>(RowKind::Insert())};
     std::vector<int64_t> levels = {KeyValue::UNKNOWN_LEVEL, KeyValue::UNKNOWN_LEVEL};
     std::vector<KeyValue> expected = KeyValueChecker::GenerateKeyValues(
-        row_kinds, {103, 105}, levels, {{4}, {6}}, {{4, 40}, {6, 60}}, pool_);
+        row_kinds, {102, 103}, levels, {{4}, {6}}, {{4, 40}, {6, 60}}, pool_);
     KeyValueChecker::CheckResult(expected, results, 1, 2);
 }
 
+TEST_F(MergedKeyValueRecordReaderTest, TestQueryRejectsNegativeOffset) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, -1, 1]])")
+            .ValueOrDie();
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        CreatePreparedQueryReaderForTest(
+            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type,
+                                                  /*read_batch_size=*/1),
+            prepared_schema, OffsetRange(1, 2), value_schema, value_schema, pool_));
+
+    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "reader offset must be non-negative");
+}
+
+TEST_F(MergedKeyValueRecordReaderTest, TestQueryOffsetCoverageAcrossReadersAndBatches) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> first_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+                                                  R"([[0, 10, 2, 1], [0, 11, 0, 2]])")
+            .ValueOrDie();
+    std::shared_ptr<arrow::Array> second_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+                                                  R"([[0, 12, 3, 3], [0, 13, 1, 4]])")
+            .ValueOrDie();
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+    batch_readers.push_back(
+        std::make_unique<MockFileBatchReader>(first_array, prepared_type, /*read_batch_size=*/1));
+    batch_readers.push_back(
+        std::make_unique<MockFileBatchReader>(second_array, prepared_type, /*read_batch_size=*/1));
+
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
+                         PreparedKeyValueReaderFactory::CreateForQuery(
+                             std::move(batch_readers), prepared_schema, OffsetRange(0, 4),
+                             value_schema, value_schema, pool_));
+    int64_t row_count = 0;
+    for (const std::unique_ptr<KeyValueRecordReader>& reader : readers) {
+        ASSERT_OK_AND_ASSIGN(
+            std::vector<KeyValue> rows,
+            (ReadResultCollector::CollectKeyValueResult<
+                KeyValueRecordReader, KeyValueRecordReader::Iterator>(reader.get())));
+        row_count += static_cast<int64_t>(rows.size());
+    }
+    ASSERT_EQ(4, row_count);
+}
+
+TEST_F(MergedKeyValueRecordReaderTest, TestQueryRejectsMissingVisibleOffset) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+                                                  R"([[0, 10, 0, 1], [0, 11, 2, 2]])")
+            .ValueOrDie();
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        CreatePreparedQueryReaderForTest(
+            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type,
+                                                  /*read_batch_size=*/1),
+            prepared_schema, OffsetRange(0, 3), value_schema, value_schema, pool_));
+
+    ASSERT_NOK_WITH_MSG(
+        (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
+                                                    KeyValueRecordReader::Iterator>(reader.get())),
+        "query readers did not cover the visible range");
+}
+
+TEST_F(MergedKeyValueRecordReaderTest, TestQueryRejectsDuplicateVisibleOffset) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> first_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1]])").ValueOrDie();
+    std::shared_ptr<arrow::Array> second_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+                                                  R"([[0, 11, 1, 2], [0, 12, 1, 3]])")
+            .ValueOrDie();
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+    batch_readers.push_back(
+        std::make_unique<MockFileBatchReader>(first_array, prepared_type, /*read_batch_size=*/1));
+    batch_readers.push_back(
+        std::make_unique<MockFileBatchReader>(second_array, prepared_type, /*read_batch_size=*/1));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
+                         PreparedKeyValueReaderFactory::CreateForQuery(
+                             std::move(batch_readers), prepared_schema, OffsetRange(0, 2),
+                             value_schema, value_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(
+        std::vector<KeyValue> first_rows,
+        (ReadResultCollector::CollectKeyValueResult<
+            KeyValueRecordReader, KeyValueRecordReader::Iterator>(readers[0].get())));
+    ASSERT_EQ(1, first_rows.size());
+    ASSERT_NOK_WITH_MSG((ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
+                                                                    KeyValueRecordReader::Iterator>(
+                            readers[1].get())),
+                        "query readers did not cover the visible range");
+}
+
+TEST_F(MergedKeyValueRecordReaderTest, TestQueryRejectsEmptyEofForVisibleRange) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([])").ValueOrDie();
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        CreatePreparedQueryReaderForTest(
+            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type,
+                                                  /*read_batch_size=*/1),
+            prepared_schema, OffsetRange(0, 1), value_schema, value_schema, pool_));
+
+    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "query readers did not cover the visible range");
+}
+
 TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderRejectsReversedVisibleOffsets) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
@@ -331,12 +472,38 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderRejectsReversedVisibleO
         std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, /*batch_size=*/1);
 
     Result<std::unique_ptr<KeyValueRecordReader>> result =
-        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, OffsetRange(2, 1),
-                                        value_schema, value_schema, pool_);
+        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
+                                         OffsetRange(2, 1), value_schema, value_schema, pool_);
     ASSERT_TRUE(result.status().IsInvalid());
     ASSERT_NOK_WITH_MSG(result, "prepared visible offset range begin exceeds end");
 }
 
+TEST_F(MergedKeyValueRecordReaderTest, TestQueryRejectsEmptyReadersForVisibleRange) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+
+    ASSERT_NOK_WITH_MSG(
+        PreparedKeyValueReaderFactory::CreateForQuery(std::move(batch_readers), prepared_schema,
+                                                      OffsetRange(0, 1), value_schema, value_schema,
+                                                      pool_),
+        "PK real-time store returned no query readers for a non-empty visible range");
+}
+
+TEST_F(MergedKeyValueRecordReaderTest, TestQueryAllowsEmptyReadersForEmptyVisibleRange) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
+                         PreparedKeyValueReaderFactory::CreateForQuery(
+                             std::move(batch_readers), prepared_schema, OffsetRange(1, 1),
+                             value_schema, value_schema, pool_));
+    ASSERT_TRUE(readers.empty());
+}
+
 TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderBitmapBounds) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
@@ -350,8 +517,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderBitmapBounds) {
 
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
-                                        value_schema, value_schema, pool_));
+        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
+                                         OffsetRange(0, 1), value_schema, value_schema, pool_));
     Result<std::vector<KeyValue>> result =
         ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
                                                    KeyValueRecordReader::Iterator>(reader.get());
@@ -359,7 +526,29 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderBitmapBounds) {
     ASSERT_NOK_WITH_MSG(result, "selected row id 1 is out of bounds for prepared batch length 1");
 }
 
-TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderCommitSchema) {
+TEST_F(MergedKeyValueRecordReaderTest, TestPreparedQueryReaderRejectsPartialBitmap) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+                                                  R"([[0, 10, 0, 1], [0, 11, 1, 2]])")
+            .ValueOrDie();
+    RoaringBitmap32 partial_bitmap;
+    partial_bitmap.Add(0);
+    auto batch_reader = std::make_unique<MockFileBatchReader>(
+        prepared_array, prepared_type, partial_bitmap, /*read_batch_size=*/2);
+    batch_reader->EnableRandomizeBatchSize(false);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
+                                         OffsetRange(0, 2), value_schema, value_schema, pool_));
+
+    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "must cover every raw mutation");
+}
+
+TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderQueryProjection) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Field> extra = MakeField("extra", arrow::int32(), 1);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
@@ -373,8 +562,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderCommitSchema) {
         std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> query_reader,
-        AdaptPreparedBatchReaderForTest(std::move(query_batch_reader), prepared_schema,
-                                        OffsetRange(0, 1), value_schema, value_schema, pool_));
+        CreatePreparedQueryReaderForTest(std::move(query_batch_reader), prepared_schema,
+                                         OffsetRange(0, 1), value_schema, value_schema, pool_));
     ASSERT_OK_AND_ASSIGN(
         std::vector<KeyValue> query_results,
         (ReadResultCollector::CollectKeyValueResult<
@@ -382,11 +571,22 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderCommitSchema) {
     ASSERT_EQ(query_results.size(), 1);
     ASSERT_EQ(query_results[0].value->GetFieldCount(), 1);
     ASSERT_EQ(query_results[0].value->GetInt(0), 1);
+}
 
+TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderRejectsNonExactCommitSchema) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Field> extra = MakeField("extra", arrow::int32(), 1);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key, extra});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1, 2]])")
+            .ValueOrDie();
     auto batch_reader = std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
+
     ASSERT_NOK_WITH_MSG(
-        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, std::nullopt,
-                                        value_schema, value_schema, pool_),
+        CreatePreparedCommitReaderForTest(std::move(batch_reader), prepared_schema,
+                                          OffsetRange(0, 1), value_schema, value_schema, pool_),
         "exact");
 }
 
@@ -424,6 +624,18 @@ TEST_F(MergedKeyValueRecordReaderTest, TestCommitOffsetCoverage) {
     ASSERT_EQ(4, row_count);
 }
 
+TEST_F(MergedKeyValueRecordReaderTest, TestCommitRejectsEmptyReaders) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+
+    ASSERT_NOK_WITH_MSG(PreparedKeyValueReaderFactory::CreateForCommit(
+                            std::move(batch_readers), prepared_schema, OffsetRange(0, 1),
+                            value_schema, value_schema, pool_),
+                        "PK real-time store returned no commit readers for a sealed segment");
+}
+
 TEST_F(MergedKeyValueRecordReaderTest, TestRejectsDuplicateCommitOffset) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
@@ -458,10 +670,10 @@ TEST_F(MergedKeyValueRecordReaderTest, TestBadCommitBatch) {
         arrow::ipc::internal::json::ArrayFromJSON(actual_type, R"([[0, 10, 0, 1]])").ValueOrDie();
 
     auto batch_reader = std::make_unique<MockFileBatchReader>(actual, actual_type, 1);
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, std::nullopt,
-                                        arrow::schema({key}), value_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
+                         CreatePreparedCommitReaderForTest(std::move(batch_reader), prepared_schema,
+                                                           OffsetRange(0, 1), arrow::schema({key}),
+                                                           value_schema, pool_));
     ASSERT_NOK_WITH_MSG(reader->NextBatch(), "field count");
 }
 
@@ -478,10 +690,10 @@ TEST_F(MergedKeyValueRecordReaderTest, TestMissingCompositeKey) {
             .ValueOrDie();
 
     auto batch_reader = std::make_unique<MockFileBatchReader>(actual, actual_type, 1);
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
-                                        arrow::schema({key0, key1}), value_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
+                         CreatePreparedQueryReaderForTest(
+                             std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
+                             arrow::schema({key0, key1}), value_schema, pool_));
     ASSERT_NOK_WITH_MSG(reader->NextBatch(), "field count");
 }
 
@@ -500,10 +712,10 @@ TEST_F(MergedKeyValueRecordReaderTest, TestQueryReaderRequiresStoreAlignedSchema
             .ValueOrDie();
 
     auto batch_reader = std::make_unique<MockFileBatchReader>(actual, actual_type, 1);
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
-                                        arrow::schema({key}), value_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
+                         CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
+                                                          OffsetRange(0, 1), arrow::schema({key}),
+                                                          value_schema, pool_));
     ASSERT_NOK_WITH_MSG(reader->NextBatch(), "field count");
 }
 
@@ -544,8 +756,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderSafeDecode) {
     auto batch_reader = std::make_unique<MockFileBatchReader>(invalid_array, invalid_type, 1);
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
-                                        value_schema, value_schema, pool_));
+        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
+                                         OffsetRange(0, 1), value_schema, value_schema, pool_));
     ASSERT_NOK_WITH_MSG(
         (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
                                                     KeyValueRecordReader::Iterator>(reader.get())),
@@ -584,8 +796,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderNestedValues) {
     auto batch_reader = std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
-                                        key_schema, query_value_schema, pool_));
+        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
+                                         OffsetRange(0, 1), key_schema, query_value_schema, pool_));
     ASSERT_OK_AND_ASSIGN(
         std::vector<KeyValue> results,
         (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
@@ -652,8 +864,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderLifecycle) {
             &explicit_close_count);
         ASSERT_OK_AND_ASSIGN(
             std::unique_ptr<KeyValueRecordReader> reader,
-            AdaptPreparedBatchReaderForTest(std::move(tracking_reader), prepared_schema,
-                                            OffsetRange(0, 1), key_schema, value_schema, pool_));
+            CreatePreparedQueryReaderForTest(std::move(tracking_reader), prepared_schema,
+                                             OffsetRange(0, 1), key_schema, value_schema, pool_));
         reader->Close();
     }
     ASSERT_EQ(explicit_close_count, 1);
@@ -665,8 +877,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderLifecycle) {
             &destructor_close_count);
         ASSERT_OK_AND_ASSIGN(
             std::unique_ptr<KeyValueRecordReader> reader,
-            AdaptPreparedBatchReaderForTest(std::move(tracking_reader), prepared_schema,
-                                            OffsetRange(0, 1), key_schema, value_schema, pool_));
+            CreatePreparedQueryReaderForTest(std::move(tracking_reader), prepared_schema,
+                                             OffsetRange(0, 1), key_schema, value_schema, pool_));
     }
     ASSERT_EQ(destructor_close_count, 1);
 
@@ -676,9 +888,9 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderLifecycle) {
             std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1),
             &factory_failure_close_count);
         std::shared_ptr<arrow::Schema> invalid_schema = arrow::schema(value_schema->fields());
-        ASSERT_NOK(AdaptPreparedBatchReaderForTest(std::move(tracking_reader), invalid_schema,
-                                                   OffsetRange(0, 1), key_schema, value_schema,
-                                                   pool_));
+        ASSERT_NOK(CreatePreparedQueryReaderForTest(std::move(tracking_reader), invalid_schema,
+                                                    OffsetRange(0, 1), key_schema, value_schema,
+                                                    pool_));
         ASSERT_EQ(nullptr, tracking_reader);
     }
     ASSERT_EQ(factory_failure_close_count, 1);
@@ -692,8 +904,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderLifecycle) {
                                                                      &read_failure_close_count);
         ASSERT_OK_AND_ASSIGN(
             std::unique_ptr<KeyValueRecordReader> reader,
-            AdaptPreparedBatchReaderForTest(std::move(tracking_reader), prepared_schema,
-                                            OffsetRange(0, 1), key_schema, value_schema, pool_));
+            CreatePreparedQueryReaderForTest(std::move(tracking_reader), prepared_schema,
+                                             OffsetRange(0, 1), key_schema, value_schema, pool_));
         ASSERT_NOK_WITH_MSG(reader->NextBatch(), "prepared reader failure");
         ASSERT_EQ(read_failure_close_count, 1);
     }
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index 8620ee872..71ec74062 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -60,25 +60,33 @@ void CloseReaders(const std::vector<std::unique_ptr<Reader>>& readers) {
 
 class RealtimeOffsetCoverage {
  public:
-    static Result<std::shared_ptr<RealtimeOffsetCoverage>> Create(const OffsetRange& sealed_offsets,
-                                                                  size_t reader_count) {
-        if (sealed_offsets.begin < 0 || sealed_offsets.end < sealed_offsets.begin) {
-            return Status::Invalid("PK real-time store returned an invalid sealed offset range");
+    static Result<std::shared_ptr<RealtimeOffsetCoverage>> Create(const OffsetRange& offsets,
+                                                                  size_t reader_count,
+                                                                  bool allow_committed_prefix) {
+        if (offsets.begin < 0 || offsets.end < offsets.begin) {
+            return Status::Invalid("PK real-time store returned an invalid offset range");
         }
         return std::shared_ptr<RealtimeOffsetCoverage>(
-            new RealtimeOffsetCoverage(sealed_offsets, reader_count));
+            new RealtimeOffsetCoverage(offsets, reader_count, allow_committed_prefix));
     }
 
     Status Add(const arrow::Int64Array& offsets) {
         for (int64_t row = 0; row < offsets.length(); ++row) {
             const int64_t offset = offsets.Value(row);
-            if (offset < sealed_offsets_.begin || offset >= sealed_offsets_.end) {
+            if (allow_committed_prefix_ && offset < 0) {
+                return Status::Invalid("PK real-time store reader offset must be non-negative");
+            }
+            if (allow_committed_prefix_ && offset < offsets_.begin) {
+                continue;
+            }
+            if (offset < offsets_.begin || offset >= offsets_.end) {
                 return Status::Invalid(
-                    "PK real-time store commit reader offset is outside the sealed range");
+                    allow_committed_prefix_
+                        ? "PK real-time store query reader offset is outside the visible range"
+                        : "PK real-time store commit reader offset is outside the sealed range");
             }
             if (!seen_offsets_.CheckedAdd(offset)) {
-                return Status::Invalid(
-                    "PK real-time store commit readers did not cover the sealed range");
+                return CoverageError();
             }
         }
         return Status::OK();
@@ -87,19 +95,29 @@ class RealtimeOffsetCoverage {
     Status FinishReader() {
         ++finished_reader_count_;
         if (finished_reader_count_ == reader_count_ &&
-            seen_offsets_.Cardinality() != sealed_offsets_.Count()) {
-            return Status::Invalid(
-                "PK real-time store commit readers did not cover the sealed range");
+            seen_offsets_.Cardinality() != offsets_.Count()) {
+            return CoverageError();
         }
         return Status::OK();
     }
 
  private:
-    RealtimeOffsetCoverage(const OffsetRange& sealed_offsets, size_t reader_count)
-        : sealed_offsets_(sealed_offsets), reader_count_(reader_count) {}
+    RealtimeOffsetCoverage(const OffsetRange& offsets, size_t reader_count,
+                           bool allow_committed_prefix)
+        : offsets_(offsets),
+          reader_count_(reader_count),
+          allow_committed_prefix_(allow_committed_prefix) {}
+
+    Status CoverageError() const {
+        return Status::Invalid(
+            allow_committed_prefix_
+                ? "PK real-time store query readers did not cover the visible range"
+                : "PK real-time store commit readers did not cover the sealed range");
+    }
 
-    OffsetRange sealed_offsets_;
+    OffsetRange offsets_;
     size_t reader_count_;
+    bool allow_committed_prefix_;
     RoaringBitmap64 seen_offsets_;
     size_t finished_reader_count_ = 0;
 };
@@ -151,6 +169,23 @@ Result<std::vector<int32_t>> ResolveFieldIndexes(
     return result;
 }
 
+Status ValidateReaderParameters(const std::shared_ptr<arrow::Schema>& prepared_schema,
+                                const std::shared_ptr<arrow::Schema>& key_schema,
+                                const std::shared_ptr<arrow::Schema>& value_schema,
+                                const std::shared_ptr<MemoryPool>& memory_pool) {
+    PAIMON_RETURN_NOT_OK(PreparedKeyValueReaderFactory::ValidateTransportSchema(prepared_schema));
+    if (!key_schema) {
+        return Status::Invalid("prepared key schema cannot be null");
+    }
+    if (!value_schema) {
+        return Status::Invalid("prepared value schema cannot be null");
+    }
+    if (!memory_pool) {
+        return Status::Invalid("prepared reader memory pool cannot be null");
+    }
+    return Status::OK();
+}
+
 Status ValidateExactCommitSchema(const std::shared_ptr<arrow::Schema>& prepared_schema,
                                  const std::shared_ptr<arrow::Schema>& value_schema) {
     if (prepared_schema->num_fields() !=
@@ -171,22 +206,7 @@ class PreparedReaderPlan {
     static Result<std::shared_ptr<const PreparedReaderPlan>> Create(
         const std::shared_ptr<arrow::Schema>& prepared_schema,
         const std::shared_ptr<arrow::Schema>& key_schema,
-        const std::shared_ptr<arrow::Schema>& value_schema,
-        const std::shared_ptr<MemoryPool>& memory_pool, bool exact_commit_schema) {
-        PAIMON_RETURN_NOT_OK(
-            PreparedKeyValueReaderFactory::ValidateTransportSchema(prepared_schema));
-        if (!key_schema) {
-            return Status::Invalid("prepared key schema cannot be null");
-        }
-        if (!value_schema) {
-            return Status::Invalid("prepared value schema cannot be null");
-        }
-        if (!memory_pool) {
-            return Status::Invalid("prepared reader memory pool cannot be null");
-        }
-        if (exact_commit_schema) {
-            PAIMON_RETURN_NOT_OK(ValidateExactCommitSchema(prepared_schema, value_schema));
-        }
+        const std::shared_ptr<arrow::Schema>& value_schema) {
         std::unordered_map<int32_t, int32_t> field_indexes;
         field_indexes.reserve(prepared_schema->num_fields() -
                               SpecialFields::kPreparedKeyValueValueStartIndex);
@@ -396,6 +416,10 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
                                 row, offsets.length()));
             }
         }
+        if (visible_offsets_.has_value() && selection.Cardinality() != offsets.length()) {
+            return Status::Invalid(
+                "PK real-time store query reader bitmap must cover every raw mutation");
+        }
         if (!visible_offsets_.has_value()) {
             selected_rows_.reserve(offsets.length());
             for (int64_t row = 0; row < offsets.length(); ++row) {
@@ -467,51 +491,17 @@ Status PreparedKeyValueReaderFactory::ValidateTransportSchema(
 
 namespace {
 
-Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderImpl(
+std::unique_ptr<KeyValueRecordReader> AdaptPreparedBatchReader(
     std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<const PreparedReaderPlan>& plan,
     const std::optional<OffsetRange>& visible_offsets,
     const std::shared_ptr<MemoryPool>& memory_pool,
     const std::shared_ptr<RealtimeOffsetCoverage>& offset_coverage) {
-    std::unique_ptr<BatchReader> owned_reader = std::move(reader);
-    if (!owned_reader) {
-        return Status::Invalid("prepared batch reader cannot be null");
-    }
-    ScopeGuard close_guard([&owned_reader]() -> void { owned_reader->Close(); });
-    std::unique_ptr<KeyValueRecordReader> result = std::make_unique<PreparedKeyValueReader>(
-        std::move(owned_reader), plan, visible_offsets, memory_pool, offset_coverage);
-    close_guard.Release();
-    return result;
+    return std::make_unique<PreparedKeyValueReader>(std::move(reader), plan, visible_offsets,
+                                                    memory_pool, offset_coverage);
 }
 
 }  // namespace
 
-Result<std::unique_ptr<KeyValueRecordReader>> PreparedKeyValueReaderFactory::Create(
-    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
-    const std::optional<OffsetRange>& visible_offsets,
-    const std::shared_ptr<arrow::Schema>& key_schema,
-    const std::shared_ptr<arrow::Schema>& value_schema,
-    const std::shared_ptr<MemoryPool>& memory_pool) {
-    std::unique_ptr<BatchReader> owned_reader = std::move(reader);
-    ScopeGuard reader_guard([&owned_reader]() {
-        if (owned_reader) {
-            owned_reader->Close();
-        }
-    });
-    if (visible_offsets.has_value() && visible_offsets->begin > visible_offsets->end) {
-        return Status::Invalid("prepared visible offset range begin exceeds end");
-    }
-    PAIMON_ASSIGN_OR_RAISE(
-        std::shared_ptr<const PreparedReaderPlan> plan,
-        PreparedReaderPlan::Create(prepared_schema, key_schema, value_schema, memory_pool,
-                                   /*exact_commit_schema=*/!visible_offsets.has_value()));
-    PAIMON_ASSIGN_OR_RAISE(
-        std::unique_ptr<KeyValueRecordReader> result,
-        AdaptPreparedBatchReaderImpl(std::move(owned_reader), plan, visible_offsets, memory_pool,
-                                     /*offset_coverage=*/nullptr));
-    reader_guard.Release();
-    return result;
-}
-
 Result<std::vector<std::unique_ptr<KeyValueRecordReader>>>
 PreparedKeyValueReaderFactory::CreateForQuery(std::vector<std::unique_ptr<BatchReader>>&& readers,
                                               const std::shared_ptr<arrow::Schema>& prepared_schema,
@@ -520,31 +510,32 @@ PreparedKeyValueReaderFactory::CreateForQuery(std::vector<std::unique_ptr<BatchR
                                               const std::shared_ptr<arrow::Schema>& value_schema,
                                               const std::shared_ptr<MemoryPool>& memory_pool) {
     std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers;
-    ScopeGuard readers_guard([&readers, &adapted_readers]() {
-        CloseReaders(readers);
-        CloseReaders(adapted_readers);
-    });
+    ScopeGuard remaining_raw_readers_guard([&readers]() { CloseReaders(readers); });
     if (visible_offsets.begin > visible_offsets.end) {
         return Status::Invalid("prepared visible offset range begin exceeds end");
     }
+    if (readers.empty() && visible_offsets.begin < visible_offsets.end) {
+        return Status::Invalid(
+            "PK real-time store returned no query readers for a non-empty visible range");
+    }
     for (const std::unique_ptr<BatchReader>& reader : readers) {
         if (!reader) {
             return Status::Invalid("PK real-time store returned a null query reader");
         }
     }
-    PAIMON_ASSIGN_OR_RAISE(
-        std::shared_ptr<const PreparedReaderPlan> plan,
-        PreparedReaderPlan::Create(prepared_schema, key_schema, value_schema, memory_pool,
-                                   /*exact_commit_schema=*/false));
+    PAIMON_RETURN_NOT_OK(
+        ValidateReaderParameters(prepared_schema, key_schema, value_schema, memory_pool));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<const PreparedReaderPlan> plan,
+                           PreparedReaderPlan::Create(prepared_schema, key_schema, value_schema));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeOffsetCoverage> offset_coverage,
+                           RealtimeOffsetCoverage::Create(visible_offsets, readers.size(),
+                                                          /*allow_committed_prefix=*/true));
     adapted_readers.reserve(readers.size());
     for (std::unique_ptr<BatchReader>& reader : readers) {
-        PAIMON_ASSIGN_OR_RAISE(
-            std::unique_ptr<KeyValueRecordReader> adapted_reader,
-            AdaptPreparedBatchReaderImpl(std::move(reader), plan, visible_offsets, memory_pool,
-                                         /*offset_coverage=*/nullptr));
-        adapted_readers.push_back(std::move(adapted_reader));
+        adapted_readers.push_back(AdaptPreparedBatchReader(std::move(reader), plan, visible_offsets,
+                                                           memory_pool, offset_coverage));
     }
-    readers_guard.Release();
+    remaining_raw_readers_guard.Release();
     return adapted_readers;
 }
 
@@ -556,29 +547,30 @@ PreparedKeyValueReaderFactory::CreateForCommit(
     const std::shared_ptr<arrow::Schema>& value_schema,
     const std::shared_ptr<MemoryPool>& memory_pool) {
     std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers;
-    ScopeGuard readers_guard([&readers, &adapted_readers]() {
-        CloseReaders(readers);
-        CloseReaders(adapted_readers);
-    });
+    ScopeGuard remaining_raw_readers_guard([&readers]() { CloseReaders(readers); });
+    if (readers.empty()) {
+        return Status::Invalid(
+            "PK real-time store returned no commit readers for a sealed segment");
+    }
     for (const std::unique_ptr<BatchReader>& reader : readers) {
         if (!reader) {
             return Status::Invalid("PK real-time store returned a null commit reader");
         }
     }
-    PAIMON_ASSIGN_OR_RAISE(
-        std::shared_ptr<const PreparedReaderPlan> plan,
-        PreparedReaderPlan::Create(prepared_schema, key_schema, value_schema, memory_pool,
-                                   /*exact_commit_schema=*/true));
+    PAIMON_RETURN_NOT_OK(
+        ValidateReaderParameters(prepared_schema, key_schema, value_schema, memory_pool));
+    PAIMON_RETURN_NOT_OK(ValidateExactCommitSchema(prepared_schema, value_schema));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<const PreparedReaderPlan> plan,
+                           PreparedReaderPlan::Create(prepared_schema, key_schema, value_schema));
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeOffsetCoverage> offset_coverage,
-                           RealtimeOffsetCoverage::Create(sealed_offsets, readers.size()));
+                           RealtimeOffsetCoverage::Create(sealed_offsets, readers.size(),
+                                                          /*allow_committed_prefix=*/false));
     adapted_readers.reserve(readers.size());
     for (std::unique_ptr<BatchReader>& reader : readers) {
-        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> adapted_reader,
-                               AdaptPreparedBatchReaderImpl(std::move(reader), plan, std::nullopt,
-                                                            memory_pool, offset_coverage));
-        adapted_readers.push_back(std::move(adapted_reader));
+        adapted_readers.push_back(AdaptPreparedBatchReader(std::move(reader), plan, std::nullopt,
+                                                           memory_pool, offset_coverage));
     }
-    readers_guard.Release();
+    remaining_raw_readers_guard.Release();
     return adapted_readers;
 }
 
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.h b/src/paimon/core/realtime/prepared_key_value_reader.h
index 658ffec08..bb03e2ad4 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.h
+++ b/src/paimon/core/realtime/prepared_key_value_reader.h
@@ -18,9 +18,7 @@
 
 #pragma once
 
-#include <cstdint>
 #include <memory>
-#include <optional>
 #include <vector>
 
 #include "arrow/type_fwd.h"
@@ -39,14 +37,6 @@ class PreparedKeyValueReaderFactory {
 
     static Status ValidateTransportSchema(const std::shared_ptr<arrow::Schema>& prepared_schema);
 
-    static Result<std::unique_ptr<KeyValueRecordReader>> Create(
-        std::unique_ptr<BatchReader>&& reader,
-        const std::shared_ptr<arrow::Schema>& prepared_schema,
-        const std::optional<OffsetRange>& visible_offsets,
-        const std::shared_ptr<arrow::Schema>& key_schema,
-        const std::shared_ptr<arrow::Schema>& value_schema,
-        const std::shared_ptr<MemoryPool>& memory_pool);
-
     static Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateForQuery(
         std::vector<std::unique_ptr<BatchReader>>&& readers,
         const std::shared_ptr<arrow::Schema>& prepared_schema, const OffsetRange& visible_offsets,
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index 73ddf050b..a8a91b4ee 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -540,21 +540,20 @@ class FailAfterPhysicalFileRealtimeStore final : public DelegatingRealtimeStore
     std::shared_ptr<std::atomic<bool>> saw_artifacts_;
 };
 
-enum class CommitReaderMalformation { DROP_LAST, DUPLICATE_OFFSET, OUT_OF_RANGE_OFFSET };
+enum class ReaderMalformation { DROP_LAST, DUPLICATE_OFFSET, OUT_OF_RANGE_OFFSET };
 
 class CorruptingBatchReader final : public BatchReader {
  public:
-    CorruptingBatchReader(std::unique_ptr<BatchReader> delegate,
-                          CommitReaderMalformation malformation)
+    CorruptingBatchReader(std::unique_ptr<BatchReader> delegate, ReaderMalformation malformation)
         : delegate_(std::move(delegate)), malformation_(malformation) {}
 
     Result<ReadBatch> NextBatch() override {
         switch (malformation_) {
-            case CommitReaderMalformation::DROP_LAST:
+            case ReaderMalformation::DROP_LAST:
                 return DropLast();
-            case CommitReaderMalformation::DUPLICATE_OFFSET:
+            case ReaderMalformation::DUPLICATE_OFFSET:
                 return SubstituteOffset(/*offset=*/0);
-            case CommitReaderMalformation::OUT_OF_RANGE_OFFSET:
+            case ReaderMalformation::OUT_OF_RANGE_OFFSET:
                 return SubstituteOffset(/*offset=*/-1);
         }
         return Status::Invalid("unknown commit reader malformation");
@@ -624,14 +623,14 @@ class CorruptingBatchReader final : public BatchReader {
     }
 
     std::unique_ptr<BatchReader> delegate_;
-    CommitReaderMalformation malformation_;
+    ReaderMalformation malformation_;
     std::optional<ReadBatch> buffered_;
 };
 
 class MalformedCoverageRealtimeStore final : public DelegatingRealtimeStore {
  public:
     MalformedCoverageRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
-                                   CommitReaderMalformation malformation)
+                                   ReaderMalformation malformation)
         : DelegatingRealtimeStore(delegate), malformation_(malformation) {}
 
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
@@ -645,7 +644,26 @@ class MalformedCoverageRealtimeStore final : public DelegatingRealtimeStore {
     }
 
  private:
-    CommitReaderMalformation malformation_;
+    ReaderMalformation malformation_;
+};
+
+class MissingQueryOffsetRealtimeStore final : public DelegatingRealtimeStore {
+ public:
+    explicit MissingQueryOffsetRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate)
+        : DelegatingRealtimeStore(delegate) {}
+
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
+        const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
+        const RealtimeQueryContext& context) override {
+        PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
+                               delegate_->CreateQueryReaders(view, offset_begin, context));
+        if (readers.empty()) {
+            return Status::Invalid("query offset drop requires a reader");
+        }
+        readers[0] = std::make_unique<CorruptingBatchReader>(std::move(readers[0]),
+                                                             ReaderMalformation::DROP_LAST);
+        return readers;
+    }
 };
 
 }  // namespace
@@ -1447,8 +1465,8 @@ class RealtimeWriteInteTest : public ::testing::Test {
         ASSERT_OK(writer->Close());
     }
 
-    void CheckPkRejectsCommitReaderMalformation(CommitReaderMalformation malformation,
-                                                const std::string& expected_error) {
+    void CheckPkRejectsReaderMalformation(ReaderMalformation malformation,
+                                          const std::string& expected_error) {
         CreatePkTable();
         auto factory = MakeDecoratingFactory<MalformedCoverageRealtimeStore>(malformation);
         ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
@@ -2385,18 +2403,34 @@ TEST_F(RealtimeWriteInteTest, TestPkMultipleStoredBatchesMergeForQueryAndCommit)
 }
 
 TEST_F(RealtimeWriteInteTest, TestPkRejectsMalformedCoverage) {
-    CheckPkRejectsCommitReaderMalformation(CommitReaderMalformation::DROP_LAST,
-                                           "commit readers did not cover the sealed range");
+    CheckPkRejectsReaderMalformation(ReaderMalformation::DROP_LAST,
+                                     "commit readers did not cover the sealed range");
 }
 
 TEST_F(RealtimeWriteInteTest, TestPkRejectsDuplicateOffset) {
-    CheckPkRejectsCommitReaderMalformation(CommitReaderMalformation::DUPLICATE_OFFSET,
-                                           "commit readers did not cover the sealed range");
+    CheckPkRejectsReaderMalformation(ReaderMalformation::DUPLICATE_OFFSET,
+                                     "commit readers did not cover the sealed range");
 }
 
 TEST_F(RealtimeWriteInteTest, TestPkRejectsOutOfRangeOffset) {
-    CheckPkRejectsCommitReaderMalformation(CommitReaderMalformation::OUT_OF_RANGE_OFFSET,
-                                           "offset is outside the sealed range");
+    CheckPkRejectsReaderMalformation(ReaderMalformation::OUT_OF_RANGE_OFFSET,
+                                     "offset is outside the sealed range");
+}
+
+TEST_F(RealtimeWriteInteTest, TestPkRejectsMissingQueryOffset) {
+    CreatePkTable();
+    auto factory = MakeDecoratingFactory<MissingQueryOffsetRealtimeStore>();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create(factory));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         MakeBatch({Row{1, "one", "p0"}, Row{2, "two", "p0"}},
+                                   /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(batch)));
+    ASSERT_NOK_WITH_MSG(ReadRows(realtime_context),
+                        "query readers did not cover the visible range");
+    ASSERT_OK(writer->Close());
 }
 
 TEST_F(RealtimeWriteInteTest, TestPkQueryReaderClose) {

From 226cd85af74b399063cf9497f7ebd7c4fef8c2c9 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Thu, 27 Aug 2026 16:39:01 +0800
Subject: [PATCH 51/62] fix(mergetree): transfer sorted reader ownership safely

---
 src/paimon/core/mergetree/merge_tree_writer.cpp     |  2 +-
 .../core/mergetree/merge_tree_writer_test.cpp       | 13 +++++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/src/paimon/core/mergetree/merge_tree_writer.cpp b/src/paimon/core/mergetree/merge_tree_writer.cpp
index bcf98e9f7..75623fd96 100644
--- a/src/paimon/core/mergetree/merge_tree_writer.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer.cpp
@@ -174,6 +174,7 @@ Status MergeTreeWriter::WriteSortedReadersToFiles(
 
     auto sort_merge_reader = std::make_unique<SortMergeReaderWithLoserTree>(
         std::move(readers), key_comparator_, user_defined_seq_comparator_, merge_function_wrapper_);
+    raw_readers_guard.Release();
     auto create_consumer = [target_schema = write_schema_, pool = pool_]()
         -> Result<std::unique_ptr<RowToArrowArrayConverter<KeyValue, KeyValueBatch>>> {
         return KeyValueMetaProjectionConsumer::Create(target_schema, pool);
@@ -181,7 +182,6 @@ Status MergeTreeWriter::WriteSortedReadersToFiles(
     auto async_key_value_producer_consumer =
         std::make_unique<AsyncKeyValueProducerAndConsumer<KeyValue, KeyValueBatch>>(
             std::move(sort_merge_reader), create_consumer, options_.GetWriteBatchSize(), 1, pool_);
-    raw_readers_guard.Release();
     ScopeGuard async_readers_guard([&]() -> void { async_key_value_producer_consumer->Close(); });
     std::unique_ptr<RollingFileWriter<KeyValueBatch, std::shared_ptr<DataFileMeta>>> rolling_writer;
     PAIMON_ASSIGN_OR_RAISE(rolling_writer, CreateRollingRowWriter());
diff --git a/src/paimon/core/mergetree/merge_tree_writer_test.cpp b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
index e93935a41..179deda44 100644
--- a/src/paimon/core/mergetree/merge_tree_writer_test.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
@@ -642,6 +642,19 @@ TEST_P(MergeTreeWriterTest, TestSortedReaderFailure) {
       [0, 0, "Alice", 10, 0, 13.1]
     ])")
             .ValueOrDie());
+    bool first_mixed_reader_closed = false;
+    bool second_mixed_reader_closed = false;
+    std::vector<std::unique_ptr<KeyValueRecordReader>> mixed_readers;
+    mixed_readers.push_back(std::make_unique<TrackingKeyValueRecordReader>(
+        CreateSingleReader(sorted_reader_array), &first_mixed_reader_closed));
+    mixed_readers.push_back(nullptr);
+    mixed_readers.push_back(std::make_unique<TrackingKeyValueRecordReader>(
+        CreateSingleReader(sorted_reader_array), &second_mixed_reader_closed));
+    Status mixed_status = merge_writer->WriteSortedReadersToFiles(std::move(mixed_readers));
+    ASSERT_TRUE(mixed_status.IsInvalid());
+    ASSERT_TRUE(first_mixed_reader_closed);
+    ASSERT_TRUE(second_mixed_reader_closed);
+
     Status expected_status = Status::IOError("sorted reader failure");
     bool failing_reader_closed = false;
     std::vector<std::unique_ptr<KeyValueRecordReader>> failing_readers;

From 3b65e105d8e66c22e94cef807b4063f38372c65d Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Thu, 27 Aug 2026 19:40:56 +0800
Subject: [PATCH 52/62] refactor(realtime): simplify reader lifecycle

---
 .../io/merged_key_value_record_reader.cpp     |  12 +-
 .../core/io/merged_key_value_record_reader.h  |   1 -
 .../merged_key_value_record_reader_test.cpp   | 166 ++----------------
 .../core/operation/merge_file_split_read.cpp  |  23 +--
 .../operation/merge_file_split_read_test.cpp  |  15 --
 .../realtime/prepared_key_value_reader.cpp    |  20 +--
 .../core/realtime/realtime_context_test.cpp   |  11 --
 test/inte/realtime_write_inte_test.cpp        |  16 +-
 8 files changed, 22 insertions(+), 242 deletions(-)

diff --git a/src/paimon/core/io/merged_key_value_record_reader.cpp b/src/paimon/core/io/merged_key_value_record_reader.cpp
index 8730086f5..70f2bcfb9 100644
--- a/src/paimon/core/io/merged_key_value_record_reader.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader.cpp
@@ -117,21 +117,13 @@ Status MergedKeyValueRecordReader::Iterator::LoadNextRawKeyValue() const {
 }
 
 Result<std::unique_ptr<KeyValueRecordReader::Iterator>> MergedKeyValueRecordReader::NextBatch() {
-    if (initialization_error_.has_value()) {
-        return initialization_error_.value();
-    }
     if (visited_) {
         return std::unique_ptr<KeyValueRecordReader::Iterator>();
     }
+    visited_ = true;
 
     auto iterator = std::make_unique<Iterator>(this);
-    Result<bool> has_next_result = iterator->HasNext();
-    if (!has_next_result.ok()) {
-        initialization_error_ = has_next_result.status();
-        return initialization_error_.value();
-    }
-    bool has_next = std::move(has_next_result).value();
-    visited_ = true;
+    PAIMON_ASSIGN_OR_RAISE(bool has_next, iterator->HasNext());
     if (!has_next) {
         return std::unique_ptr<KeyValueRecordReader::Iterator>();
     }
diff --git a/src/paimon/core/io/merged_key_value_record_reader.h b/src/paimon/core/io/merged_key_value_record_reader.h
index 227a1593a..a1b7aa5e4 100644
--- a/src/paimon/core/io/merged_key_value_record_reader.h
+++ b/src/paimon/core/io/merged_key_value_record_reader.h
@@ -67,7 +67,6 @@ class MergedKeyValueRecordReader : public KeyValueRecordReader {
 
  private:
     bool visited_ = false;
-    std::optional<Status> initialization_error_;
     std::unique_ptr<KeyValueRecordReader> reader_;
     std::shared_ptr<FieldsComparator> key_comparator_;
     std::shared_ptr<MergeFunctionWrapper<KeyValue>> merge_function_wrapper_;
diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index 8cdef7a6b..d61428aa4 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -146,54 +146,6 @@ class MalformedBitmapBatchReader : public BatchReader {
     int32_t row_id_;
 };
 
-class ScriptedKeyValueRecordReader final : public KeyValueRecordReader {
- public:
-    ScriptedKeyValueRecordReader(std::vector<KeyValue>&& key_values, int32_t* next_batch_count)
-        : key_values_(std::move(key_values)), next_batch_count_(next_batch_count) {}
-
-    class Iterator final : public KeyValueRecordReader::Iterator {
-     public:
-        explicit Iterator(KeyValue&& key_value) : key_value_(std::move(key_value)) {}
-
-        Result<bool> HasNext() const override {
-            return key_value_.has_value();
-        }
-
-        Result<KeyValue> Next() override {
-            KeyValue result = std::move(key_value_.value());
-            key_value_.reset();
-            return result;
-        }
-
-     private:
-        std::optional<KeyValue> key_value_;
-    };
-
-    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatch() override {
-        ++(*next_batch_count_);
-        if (*next_batch_count_ == 1) {
-            return std::make_unique<Iterator>(std::move(key_values_[0]));
-        }
-        if (*next_batch_count_ == 2) {
-            return Status::IOError("scripted lookahead failure");
-        }
-        if (*next_batch_count_ == 3) {
-            return std::make_unique<Iterator>(std::move(key_values_[1]));
-        }
-        return std::unique_ptr<KeyValueRecordReader::Iterator>();
-    }
-
-    std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return nullptr;
-    }
-
-    void Close() override {}
-
- private:
-    std::vector<KeyValue> key_values_;
-    int32_t* next_batch_count_;
-};
-
 }  // namespace
 
 class MergedKeyValueRecordReaderTest : public testing::Test {
@@ -461,23 +413,6 @@ TEST_F(MergedKeyValueRecordReaderTest, TestQueryRejectsEmptyEofForVisibleRange)
     ASSERT_NOK_WITH_MSG(reader->NextBatch(), "query readers did not cover the visible range");
 }
 
-TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderRejectsReversedVisibleOffsets) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1]])").ValueOrDie();
-    auto batch_reader =
-        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, /*batch_size=*/1);
-
-    Result<std::unique_ptr<KeyValueRecordReader>> result =
-        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
-                                         OffsetRange(2, 1), value_schema, value_schema, pool_);
-    ASSERT_TRUE(result.status().IsInvalid());
-    ASSERT_NOK_WITH_MSG(result, "prepared visible offset range begin exceeds end");
-}
-
 TEST_F(MergedKeyValueRecordReaderTest, TestQueryRejectsEmptyReadersForVisibleRange) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
@@ -573,23 +508,6 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderQueryProjection) {
     ASSERT_EQ(query_results[0].value->GetInt(0), 1);
 }
 
-TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderRejectsNonExactCommitSchema) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Field> extra = MakeField("extra", arrow::int32(), 1);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key, extra});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1, 2]])")
-            .ValueOrDie();
-    auto batch_reader = std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
-
-    ASSERT_NOK_WITH_MSG(
-        CreatePreparedCommitReaderForTest(std::move(batch_reader), prepared_schema,
-                                          OffsetRange(0, 1), value_schema, value_schema, pool_),
-        "exact");
-}
-
 TEST_F(MergedKeyValueRecordReaderTest, TestCommitOffsetCoverage) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
@@ -719,28 +637,6 @@ TEST_F(MergedKeyValueRecordReaderTest, TestQueryReaderRequiresStoreAlignedSchema
     ASSERT_NOK_WITH_MSG(reader->NextBatch(), "field count");
 }
 
-TEST_F(MergedKeyValueRecordReaderTest, TestMergedReaderInitializationErrorIsTerminal) {
-    std::vector<DataField> key_fields = {DataField(0, arrow::field("key", arrow::int32()))};
-    std::vector<KeyValue> key_values =
-        KeyValueChecker::GenerateKeyValues({10, 11}, {{1}, {2}}, {{1}, {2}}, pool_);
-    int32_t next_batch_count = 0;
-    auto reader =
-        std::make_unique<ScriptedKeyValueRecordReader>(std::move(key_values), &next_batch_count);
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<FieldsComparator> key_comparator,
-                         FieldsComparator::Create(key_fields, true));
-    MergedKeyValueRecordReader merged_reader(std::move(reader), key_comparator,
-                                             merge_function_wrapper_);
-
-    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> first = merged_reader.NextBatch();
-    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> retry = merged_reader.NextBatch();
-    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> second_retry =
-        merged_reader.NextBatch();
-    ASSERT_NOK_WITH_MSG(first, "scripted lookahead failure");
-    ASSERT_EQ(first.status().ToString(), retry.status().ToString());
-    ASSERT_EQ(first.status().ToString(), second_retry.status().ToString());
-    ASSERT_EQ(2, next_batch_count);
-}
-
 TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderSafeDecode) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
@@ -843,7 +739,7 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderNestedValues) {
     ASSERT_EQ(keyed_value_map->ValueArray()->GetInt(1), 23);
 }
 
-TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderLifecycle) {
+TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderFactoryFailureClosesReaders) {
     std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32())),
                                            DataField(1, arrow::field("v0", arrow::int32()))};
     std::shared_ptr<arrow::Schema> value_schema =
@@ -857,59 +753,17 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderLifecycle) {
     ])")
             .ValueOrDie());
 
-    int32_t explicit_close_count = 0;
-    {
-        auto tracking_reader = std::make_unique<TrackingBatchReader>(
-            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1),
-            &explicit_close_count);
-        ASSERT_OK_AND_ASSIGN(
-            std::unique_ptr<KeyValueRecordReader> reader,
-            CreatePreparedQueryReaderForTest(std::move(tracking_reader), prepared_schema,
-                                             OffsetRange(0, 1), key_schema, value_schema, pool_));
-        reader->Close();
-    }
-    ASSERT_EQ(explicit_close_count, 1);
-
-    int32_t destructor_close_count = 0;
-    {
-        auto tracking_reader = std::make_unique<TrackingBatchReader>(
-            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1),
-            &destructor_close_count);
-        ASSERT_OK_AND_ASSIGN(
-            std::unique_ptr<KeyValueRecordReader> reader,
-            CreatePreparedQueryReaderForTest(std::move(tracking_reader), prepared_schema,
-                                             OffsetRange(0, 1), key_schema, value_schema, pool_));
-    }
-    ASSERT_EQ(destructor_close_count, 1);
-
     int32_t factory_failure_close_count = 0;
-    {
-        std::unique_ptr<BatchReader> tracking_reader = std::make_unique<TrackingBatchReader>(
-            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1),
-            &factory_failure_close_count);
-        std::shared_ptr<arrow::Schema> invalid_schema = arrow::schema(value_schema->fields());
-        ASSERT_NOK(CreatePreparedQueryReaderForTest(std::move(tracking_reader), invalid_schema,
-                                                    OffsetRange(0, 1), key_schema, value_schema,
-                                                    pool_));
-        ASSERT_EQ(nullptr, tracking_reader);
-    }
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+    batch_readers.push_back(std::make_unique<TrackingBatchReader>(
+        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1),
+        &factory_failure_close_count));
+    batch_readers.push_back(nullptr);
+    ASSERT_NOK_WITH_MSG(PreparedKeyValueReaderFactory::CreateForQuery(
+                            std::move(batch_readers), prepared_schema, OffsetRange(0, 1),
+                            key_schema, value_schema, pool_),
+                        "PK real-time store returned a null query reader");
     ASSERT_EQ(factory_failure_close_count, 1);
-
-    int32_t read_failure_close_count = 0;
-    {
-        auto failing_reader =
-            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
-        failing_reader->SetNextBatchStatus(Status::IOError("prepared reader failure"));
-        auto tracking_reader = std::make_unique<TrackingBatchReader>(std::move(failing_reader),
-                                                                     &read_failure_close_count);
-        ASSERT_OK_AND_ASSIGN(
-            std::unique_ptr<KeyValueRecordReader> reader,
-            CreatePreparedQueryReaderForTest(std::move(tracking_reader), prepared_schema,
-                                             OffsetRange(0, 1), key_schema, value_schema, pool_));
-        ASSERT_NOK_WITH_MSG(reader->NextBatch(), "prepared reader failure");
-        ASSERT_EQ(read_failure_close_count, 1);
-    }
-    ASSERT_EQ(read_failure_close_count, 1);
 }
 
 }  // namespace paimon::test
diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp
index 3b09d8c54..835ed0932 100644
--- a/src/paimon/core/operation/merge_file_split_read.cpp
+++ b/src/paimon/core/operation/merge_file_split_read.cpp
@@ -149,28 +149,17 @@ class MergeFileSplitRead::RealtimeReaderBuilder {
 
     Status CollectDiskReaders(const std::vector<std::shared_ptr<Split>>& disk_splits,
                               std::vector<std::unique_ptr<KeyValueRecordReader>>* readers) {
-        std::vector<std::shared_ptr<DataSplitImpl>> data_splits;
-        data_splits.reserve(disk_splits.size());
+        std::shared_ptr<DataSplitImpl> first_split;
+        std::vector<std::shared_ptr<DataFileMeta>> data_files;
+        std::vector<std::optional<DeletionFile>> deletion_files;
         for (const std::shared_ptr<Split>& disk_split : disk_splits) {
             std::shared_ptr<DataSplitImpl> data_split =
                 std::dynamic_pointer_cast<DataSplitImpl>(disk_split);
             if (!data_split) {
                 return Status::Invalid("merge input disk split is not a data split");
             }
-            data_splits.push_back(std::move(data_split));
-        }
-        const std::shared_ptr<DataSplitImpl>& first_split = data_splits.front();
-        const BinaryRow& partition = first_split->Partition();
-        const int32_t bucket = first_split->Bucket();
-
-        std::vector<std::shared_ptr<DataFileMeta>> data_files;
-        std::vector<std::optional<DeletionFile>> deletion_files;
-        for (const std::shared_ptr<DataSplitImpl>& data_split : data_splits) {
-            if (!(data_split->Partition() == partition) || data_split->Bucket() != bucket) {
-                return Status::Invalid("merge input disk splits do not share a partition-bucket");
-            }
-            if (!data_split->BeforeFiles().empty()) {
-                return Status::Invalid("merge input disk split must not contain before files");
+            if (!first_split) {
+                first_split = data_split;
             }
             const std::vector<std::shared_ptr<DataFileMeta>>& split_files = data_split->DataFiles();
             const std::vector<std::optional<DeletionFile>>& split_deletion_files =
@@ -188,6 +177,8 @@ class MergeFileSplitRead::RealtimeReaderBuilder {
                                       split_deletion_files.end());
             }
         }
+        const BinaryRow& partition = first_split->Partition();
+        const int32_t bucket = first_split->Bucket();
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<DataFilePathFactory> data_file_path_factory,
                                owner_->path_factory_->CreateDataFilePathFactory(partition, bucket));
 
diff --git a/src/paimon/core/operation/merge_file_split_read_test.cpp b/src/paimon/core/operation/merge_file_split_read_test.cpp
index d1da3e4f6..7a859cae5 100644
--- a/src/paimon/core/operation/merge_file_split_read_test.cpp
+++ b/src/paimon/core/operation/merge_file_split_read_test.cpp
@@ -766,21 +766,6 @@ TEST_P(MergeFileSplitReadTest, TestRealtimeReadValidatesDiskSplits) {
     std::vector<std::shared_ptr<Split>> non_data_splits = {std::make_shared<TestingSplit>()};
     ASSERT_NOK_WITH_MSG(split_read->CreateRealtimeReader(non_data_splits, {}),
                         "merge input disk split is not a data split");
-    std::vector<std::shared_ptr<Split>> mixed_partition_splits = {prepared_splits[0],
-                                                                 prepared_splits[1]};
-    ASSERT_NOK_WITH_MSG(split_read->CreateRealtimeReader(mixed_partition_splits, {}),
-                        "merge input disk splits do not share a partition-bucket");
-
-    std::vector<std::shared_ptr<DataFileMeta>> before_data_files = first->DataFiles();
-    DataSplitImpl::Builder before_builder(first->Partition(), first->Bucket(), first->BucketPath(),
-                                          std::move(before_data_files));
-    std::vector<std::shared_ptr<DataFileMeta>> before_files = first->DataFiles();
-    ASSERT_OK_AND_ASSIGN(
-        std::shared_ptr<DataSplitImpl> before_split,
-        before_builder.WithBeforeFiles(std::move(before_files)).RawConvertible(false).Build());
-    std::vector<std::shared_ptr<Split>> before_splits = {before_split};
-    ASSERT_NOK_WITH_MSG(split_read->CreateRealtimeReader(before_splits, {}),
-                        "merge input disk split must not contain before files");
 
     std::vector<std::shared_ptr<DataFileMeta>> deletion_data_files = first->DataFiles();
     DataSplitImpl::Builder deletion_builder(first->Partition(), first->Bucket(),
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index 71ec74062..c2d729900 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -264,10 +264,6 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
           pool_(pool),
           offset_coverage_(offset_coverage) {}
 
-    ~PreparedKeyValueReader() override {
-        Close();
-    }
-
     class Iterator final : public KeyValueRecordReader::Iterator {
      public:
         explicit Iterator(PreparedKeyValueReader* reader) : reader_(reader) {}
@@ -298,15 +294,7 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
     };
 
     Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatch() override {
-        if (first_error_.has_value()) {
-            return first_error_.value();
-        }
-        Result<std::unique_ptr<KeyValueRecordReader::Iterator>> result = NextBatchImpl();
-        if (!result.ok()) {
-            first_error_ = result.status();
-            Close();
-        }
-        return result;
+        return NextBatchImpl();
     }
 
     std::shared_ptr<Metrics> GetReaderMetrics() const override {
@@ -314,10 +302,6 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
     }
 
     void Close() override {
-        if (closed_) {
-            return;
-        }
-        closed_ = true;
         ResetBatchState();
         reader_->Close();
     }
@@ -454,8 +438,6 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
     }
 
  private:
-    bool closed_ = false;
-    std::optional<Status> first_error_;
     std::unique_ptr<BatchReader> reader_;
     std::shared_ptr<const PreparedReaderPlan> plan_;
     std::optional<OffsetRange> visible_offsets_;
diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp
index 6dd1d7577..82f831f6a 100644
--- a/src/paimon/core/realtime/realtime_context_test.cpp
+++ b/src/paimon/core/realtime/realtime_context_test.cpp
@@ -234,17 +234,6 @@ TEST(RealtimeContextTest, TestReconcilesPrimaryKeyInitialSequence) {
     ASSERT_EQ(10, fourth);
 }
 
-TEST(RealtimeContextTest, TestMaterializedSequenceRejectsMissingStore) {
-    auto factory = std::make_shared<TestingRealtimeStoreFactory>();
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
-
-    Result<int64_t> result = context->AdvanceMaterializedMaxSequenceNumber(
-        RealtimePartitionBucket({{"dt", "missing"}}, /*bucket=*/3),
-        /*max_sequence_number=*/4);
-    ASSERT_TRUE(result.status().IsKeyError());
-    ASSERT_NOK_WITH_MSG(result, "real-time store not found for partition {dt=missing}, bucket 3");
-}
-
 TEST(RealtimeContextTest, TestCommittedProgressIsMonotonicAndSelective) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index a8a91b4ee..73391b1a6 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -2445,21 +2445,9 @@ TEST_F(RealtimeWriteInteTest, TestPkQueryReaderClose) {
                          MakeBatch({Row{1, "one", "p0"}}, /*partitioned=*/false));
     ASSERT_OK(writer->Write(std::move(batch)));
 
-    auto release_reader = [&](bool explicit_close) -> Status {
-        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> reader,
-                               CreateQueryReader(realtime_context));
-        if (explicit_close) {
-            reader->Close();
-        }
-        return Status::OK();
-    };
-
-    ASSERT_OK(release_reader(/*explicit_close=*/true));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> reader, CreateQueryReader(realtime_context));
+    reader->Close();
     ASSERT_EQ(1, state->query_close_count->load(std::memory_order_acquire));
-
-    ASSERT_OK(release_reader(/*explicit_close=*/false));
-    ASSERT_EQ(2, state->query_close_count->load(std::memory_order_acquire));
-
     ASSERT_OK(writer->Close());
 }
 

From e8a5b2913e9518b83207064135bc21e5fe5982a1 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Thu, 27 Aug 2026 21:27:58 +0800
Subject: [PATCH 53/62] refactor(realtime): simplify prepared reader validation

---
 src/paimon/CMakeLists.txt                     |   1 +
 .../merged_key_value_record_reader_test.cpp   | 616 -----------------
 .../realtime/prepared_key_value_reader.cpp    |  63 +-
 .../prepared_key_value_reader_test.cpp        | 621 ++++++++++++++++++
 4 files changed, 628 insertions(+), 673 deletions(-)
 create mode 100644 src/paimon/core/realtime/prepared_key_value_reader_test.cpp

diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index 0a78b0902..41fe5f6ba 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -791,6 +791,7 @@ if(PAIMON_BUILD_TESTS)
                     core/memory/writer_memory_manager_test.cpp
                     core/realtime/arrow_realtime_store_test.cpp
                     core/realtime/primary_key_realtime_store_test.cpp
+                    core/realtime/prepared_key_value_reader_test.cpp
                     core/realtime/realtime_context_test.cpp
                     core/realtime/realtime_reader_test.cpp
                     core/mergetree/levels_test.cpp
diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index d61428aa4..a9395c9ab 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -45,109 +45,6 @@
 #include "paimon/testing/utils/testharness.h"
 
 namespace paimon::test {
-
-namespace {
-
-std::shared_ptr<arrow::Field> MakeField(const std::string& name,
-                                        const std::shared_ptr<arrow::DataType>& type,
-                                        int32_t field_id, bool nullable = true) {
-    return DataField::ConvertDataFieldToArrowField(
-        DataField(field_id, arrow::field(name, type, nullable)));
-}
-
-std::shared_ptr<arrow::Schema> MakePreparedSchema(const arrow::FieldVector& value_fields) {
-    arrow::FieldVector prepared_fields = {
-        DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
-        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
-            ->WithNullable(false),
-        DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset())};
-    prepared_fields.insert(prepared_fields.end(), value_fields.begin(), value_fields.end());
-    return arrow::schema(prepared_fields);
-}
-
-Result<std::unique_ptr<KeyValueRecordReader>> CreatePreparedQueryReaderForTest(
-    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
-    const OffsetRange& visible_offsets, const std::shared_ptr<arrow::Schema>& key_schema,
-    const std::shared_ptr<arrow::Schema>& value_schema,
-    const std::shared_ptr<MemoryPool>& memory_pool) {
-    std::vector<std::unique_ptr<BatchReader>> readers;
-    readers.push_back(std::move(reader));
-    PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers,
-                           PreparedKeyValueReaderFactory::CreateForQuery(
-                               std::move(readers), prepared_schema, visible_offsets, key_schema,
-                               value_schema, memory_pool));
-    return std::move(adapted_readers[0]);
-}
-
-Result<std::unique_ptr<KeyValueRecordReader>> CreatePreparedCommitReaderForTest(
-    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
-    const OffsetRange& sealed_offsets, const std::shared_ptr<arrow::Schema>& key_schema,
-    const std::shared_ptr<arrow::Schema>& value_schema,
-    const std::shared_ptr<MemoryPool>& memory_pool) {
-    std::vector<std::unique_ptr<BatchReader>> readers;
-    readers.push_back(std::move(reader));
-    PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers,
-                           PreparedKeyValueReaderFactory::CreateForCommit(
-                               std::move(readers), prepared_schema, sealed_offsets, key_schema,
-                               value_schema, memory_pool));
-    return std::move(adapted_readers[0]);
-}
-
-class TrackingBatchReader : public BatchReader {
- public:
-    TrackingBatchReader(std::unique_ptr<BatchReader>&& delegate, int32_t* close_count)
-        : delegate_(std::move(delegate)), close_count_(close_count) {}
-
-    Result<ReadBatch> NextBatch() override {
-        return delegate_->NextBatch();
-    }
-
-    std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return delegate_->GetReaderMetrics();
-    }
-
-    void Close() override {
-        ++(*close_count_);
-        delegate_->Close();
-    }
-
- private:
-    std::unique_ptr<BatchReader> delegate_;
-    int32_t* close_count_;
-};
-
-class MalformedBitmapBatchReader : public BatchReader {
- public:
-    MalformedBitmapBatchReader(std::unique_ptr<BatchReader>&& delegate, int32_t row_id)
-        : delegate_(std::move(delegate)), row_id_(row_id) {}
-
-    Result<ReadBatch> NextBatch() override {
-        return delegate_->NextBatch();
-    }
-
-    Result<ReadBatchWithBitmap> NextBatchWithBitmap() override {
-        PAIMON_ASSIGN_OR_RAISE(ReadBatchWithBitmap batch, delegate_->NextBatchWithBitmap());
-        if (!IsEofBatch(batch)) {
-            batch.second.Add(row_id_);
-        }
-        return batch;
-    }
-
-    std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return delegate_->GetReaderMetrics();
-    }
-
-    void Close() override {
-        delegate_->Close();
-    }
-
- private:
-    std::unique_ptr<BatchReader> delegate_;
-    int32_t row_id_;
-};
-
-}  // namespace
-
 class MergedKeyValueRecordReaderTest : public testing::Test {
  public:
     void SetUp() override {
@@ -253,517 +150,4 @@ TEST_F(MergedKeyValueRecordReaderTest, TestSkipMergedNulloptResultInHasNext) {
     }
 }
 
-TEST_F(MergedKeyValueRecordReaderTest, TestQueryAllowsCommittedPrefix) {
-    std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32())),
-                                           DataField(1, arrow::field("v0", arrow::int32()))};
-    std::shared_ptr<arrow::Schema> value_schema =
-        DataField::ConvertDataFieldsToArrowSchema(value_fields);
-    std::shared_ptr<arrow::Schema> key_schema = arrow::schema({value_schema->field(0)});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema(value_schema->fields());
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
-        [0, 100, 0, 1, 10],
-        [0, 101, 1, 2, 20],
-        [0, 102, 2, 4, 40],
-        [0, 103, 3, 6, 60]
-    ])")
-            .ValueOrDie());
-
-    std::vector<std::unique_ptr<BatchReader>> batch_readers;
-    batch_readers.push_back(
-        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 2));
-    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
-                         PreparedKeyValueReaderFactory::CreateForQuery(
-                             std::move(batch_readers), prepared_schema, OffsetRange(2, 4),
-                             key_schema, value_schema, pool_));
-    ASSERT_EQ(1, readers.size());
-    ASSERT_OK_AND_ASSIGN(
-        std::vector<KeyValue> results,
-        (ReadResultCollector::CollectKeyValueResult<
-            KeyValueRecordReader, KeyValueRecordReader::Iterator>(readers[0].get())));
-
-    std::vector<RowKind*> row_kinds = {const_cast<RowKind*>(RowKind::Insert()),
-                                       const_cast<RowKind*>(RowKind::Insert())};
-    std::vector<int64_t> levels = {KeyValue::UNKNOWN_LEVEL, KeyValue::UNKNOWN_LEVEL};
-    std::vector<KeyValue> expected = KeyValueChecker::GenerateKeyValues(
-        row_kinds, {102, 103}, levels, {{4}, {6}}, {{4, 40}, {6, 60}}, pool_);
-    KeyValueChecker::CheckResult(expected, results, 1, 2);
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestQueryRejectsNegativeOffset) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, -1, 1]])")
-            .ValueOrDie();
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        CreatePreparedQueryReaderForTest(
-            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type,
-                                                  /*read_batch_size=*/1),
-            prepared_schema, OffsetRange(1, 2), value_schema, value_schema, pool_));
-
-    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "reader offset must be non-negative");
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestQueryOffsetCoverageAcrossReadersAndBatches) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> first_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
-                                                  R"([[0, 10, 2, 1], [0, 11, 0, 2]])")
-            .ValueOrDie();
-    std::shared_ptr<arrow::Array> second_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
-                                                  R"([[0, 12, 3, 3], [0, 13, 1, 4]])")
-            .ValueOrDie();
-    std::vector<std::unique_ptr<BatchReader>> batch_readers;
-    batch_readers.push_back(
-        std::make_unique<MockFileBatchReader>(first_array, prepared_type, /*read_batch_size=*/1));
-    batch_readers.push_back(
-        std::make_unique<MockFileBatchReader>(second_array, prepared_type, /*read_batch_size=*/1));
-
-    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
-                         PreparedKeyValueReaderFactory::CreateForQuery(
-                             std::move(batch_readers), prepared_schema, OffsetRange(0, 4),
-                             value_schema, value_schema, pool_));
-    int64_t row_count = 0;
-    for (const std::unique_ptr<KeyValueRecordReader>& reader : readers) {
-        ASSERT_OK_AND_ASSIGN(
-            std::vector<KeyValue> rows,
-            (ReadResultCollector::CollectKeyValueResult<
-                KeyValueRecordReader, KeyValueRecordReader::Iterator>(reader.get())));
-        row_count += static_cast<int64_t>(rows.size());
-    }
-    ASSERT_EQ(4, row_count);
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestQueryRejectsMissingVisibleOffset) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
-                                                  R"([[0, 10, 0, 1], [0, 11, 2, 2]])")
-            .ValueOrDie();
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        CreatePreparedQueryReaderForTest(
-            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type,
-                                                  /*read_batch_size=*/1),
-            prepared_schema, OffsetRange(0, 3), value_schema, value_schema, pool_));
-
-    ASSERT_NOK_WITH_MSG(
-        (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
-                                                    KeyValueRecordReader::Iterator>(reader.get())),
-        "query readers did not cover the visible range");
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestQueryRejectsDuplicateVisibleOffset) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> first_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1]])").ValueOrDie();
-    std::shared_ptr<arrow::Array> second_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
-                                                  R"([[0, 11, 1, 2], [0, 12, 1, 3]])")
-            .ValueOrDie();
-    std::vector<std::unique_ptr<BatchReader>> batch_readers;
-    batch_readers.push_back(
-        std::make_unique<MockFileBatchReader>(first_array, prepared_type, /*read_batch_size=*/1));
-    batch_readers.push_back(
-        std::make_unique<MockFileBatchReader>(second_array, prepared_type, /*read_batch_size=*/1));
-    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
-                         PreparedKeyValueReaderFactory::CreateForQuery(
-                             std::move(batch_readers), prepared_schema, OffsetRange(0, 2),
-                             value_schema, value_schema, pool_));
-    ASSERT_OK_AND_ASSIGN(
-        std::vector<KeyValue> first_rows,
-        (ReadResultCollector::CollectKeyValueResult<
-            KeyValueRecordReader, KeyValueRecordReader::Iterator>(readers[0].get())));
-    ASSERT_EQ(1, first_rows.size());
-    ASSERT_NOK_WITH_MSG((ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
-                                                                    KeyValueRecordReader::Iterator>(
-                            readers[1].get())),
-                        "query readers did not cover the visible range");
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestQueryRejectsEmptyEofForVisibleRange) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([])").ValueOrDie();
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        CreatePreparedQueryReaderForTest(
-            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type,
-                                                  /*read_batch_size=*/1),
-            prepared_schema, OffsetRange(0, 1), value_schema, value_schema, pool_));
-
-    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "query readers did not cover the visible range");
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestQueryRejectsEmptyReadersForVisibleRange) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::vector<std::unique_ptr<BatchReader>> batch_readers;
-
-    ASSERT_NOK_WITH_MSG(
-        PreparedKeyValueReaderFactory::CreateForQuery(std::move(batch_readers), prepared_schema,
-                                                      OffsetRange(0, 1), value_schema, value_schema,
-                                                      pool_),
-        "PK real-time store returned no query readers for a non-empty visible range");
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestQueryAllowsEmptyReadersForEmptyVisibleRange) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::vector<std::unique_ptr<BatchReader>> batch_readers;
-
-    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
-                         PreparedKeyValueReaderFactory::CreateForQuery(
-                             std::move(batch_readers), prepared_schema, OffsetRange(1, 1),
-                             value_schema, value_schema, pool_));
-    ASSERT_TRUE(readers.empty());
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderBitmapBounds) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1]])").ValueOrDie();
-    auto batch_reader = std::make_unique<MalformedBitmapBatchReader>(
-        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, /*batch_size=*/1),
-        /*row_id=*/1);
-
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
-                                         OffsetRange(0, 1), value_schema, value_schema, pool_));
-    Result<std::vector<KeyValue>> result =
-        ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
-                                                   KeyValueRecordReader::Iterator>(reader.get());
-    ASSERT_TRUE(result.status().IsInvalid());
-    ASSERT_NOK_WITH_MSG(result, "selected row id 1 is out of bounds for prepared batch length 1");
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestPreparedQueryReaderRejectsPartialBitmap) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
-                                                  R"([[0, 10, 0, 1], [0, 11, 1, 2]])")
-            .ValueOrDie();
-    RoaringBitmap32 partial_bitmap;
-    partial_bitmap.Add(0);
-    auto batch_reader = std::make_unique<MockFileBatchReader>(
-        prepared_array, prepared_type, partial_bitmap, /*read_batch_size=*/2);
-    batch_reader->EnableRandomizeBatchSize(false);
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
-                                         OffsetRange(0, 2), value_schema, value_schema, pool_));
-
-    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "must cover every raw mutation");
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderQueryProjection) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Field> extra = MakeField("extra", arrow::int32(), 1);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key, extra});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1, 2]])")
-            .ValueOrDie());
-
-    auto query_batch_reader =
-        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> query_reader,
-        CreatePreparedQueryReaderForTest(std::move(query_batch_reader), prepared_schema,
-                                         OffsetRange(0, 1), value_schema, value_schema, pool_));
-    ASSERT_OK_AND_ASSIGN(
-        std::vector<KeyValue> query_results,
-        (ReadResultCollector::CollectKeyValueResult<
-            KeyValueRecordReader, KeyValueRecordReader::Iterator>(query_reader.get())));
-    ASSERT_EQ(query_results.size(), 1);
-    ASSERT_EQ(query_results[0].value->GetFieldCount(), 1);
-    ASSERT_EQ(query_results[0].value->GetInt(0), 1);
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestCommitOffsetCoverage) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> first_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
-                                                  R"([[0, 10, 2, 1], [0, 11, 0, 3]])")
-            .ValueOrDie();
-    std::shared_ptr<arrow::Array> second_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
-                                                  R"([[0, 12, 1, 2], [0, 13, 3, 4]])")
-            .ValueOrDie();
-    std::vector<std::unique_ptr<BatchReader>> batch_readers;
-    batch_readers.push_back(
-        std::make_unique<MockFileBatchReader>(first_array, prepared_type, /*read_batch_size=*/1));
-    batch_readers.push_back(
-        std::make_unique<MockFileBatchReader>(second_array, prepared_type, /*read_batch_size=*/1));
-
-    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
-                         PreparedKeyValueReaderFactory::CreateForCommit(
-                             std::move(batch_readers), prepared_schema, OffsetRange(0, 4),
-                             value_schema, value_schema, pool_));
-    int64_t row_count = 0;
-    for (const std::unique_ptr<KeyValueRecordReader>& reader : readers) {
-        ASSERT_OK_AND_ASSIGN(
-            std::vector<KeyValue> rows,
-            (ReadResultCollector::CollectKeyValueResult<
-                KeyValueRecordReader, KeyValueRecordReader::Iterator>(reader.get())));
-        row_count += static_cast<int64_t>(rows.size());
-    }
-    ASSERT_EQ(4, row_count);
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestCommitRejectsEmptyReaders) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::vector<std::unique_ptr<BatchReader>> batch_readers;
-
-    ASSERT_NOK_WITH_MSG(PreparedKeyValueReaderFactory::CreateForCommit(
-                            std::move(batch_readers), prepared_schema, OffsetRange(0, 1),
-                            value_schema, value_schema, pool_),
-                        "PK real-time store returned no commit readers for a sealed segment");
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestRejectsDuplicateCommitOffset) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
-        arrow::ipc::internal::json::ArrayFromJSON(
-            prepared_type, R"([[0, 10, 0, 1], [0, 11, 0, 2], [0, 12, 2, 3]])")
-            .ValueOrDie();
-    std::vector<std::unique_ptr<BatchReader>> batch_readers;
-    batch_readers.push_back(std::make_unique<MockFileBatchReader>(prepared_array, prepared_type,
-                                                                  /*read_batch_size=*/1));
-
-    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
-                         PreparedKeyValueReaderFactory::CreateForCommit(
-                             std::move(batch_readers), prepared_schema, OffsetRange(0, 3),
-                             value_schema, value_schema, pool_));
-    ASSERT_NOK_WITH_MSG((ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
-                                                                    KeyValueRecordReader::Iterator>(
-                            readers[0].get())),
-                        "did not cover the sealed range");
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestBadCommitBatch) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Field> value = MakeField("value", arrow::int32(), 1);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key, value});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key, value});
-    std::shared_ptr<arrow::Schema> actual_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> actual_type = arrow::struct_(actual_schema->fields());
-    std::shared_ptr<arrow::Array> actual =
-        arrow::ipc::internal::json::ArrayFromJSON(actual_type, R"([[0, 10, 0, 1]])").ValueOrDie();
-
-    auto batch_reader = std::make_unique<MockFileBatchReader>(actual, actual_type, 1);
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
-                         CreatePreparedCommitReaderForTest(std::move(batch_reader), prepared_schema,
-                                                           OffsetRange(0, 1), arrow::schema({key}),
-                                                           value_schema, pool_));
-    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "field count");
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestMissingCompositeKey) {
-    std::shared_ptr<arrow::Field> key0 = MakeField("key0", arrow::int32(), 0);
-    std::shared_ptr<arrow::Field> key1 = MakeField("key1", arrow::int32(), 1);
-    std::shared_ptr<arrow::Field> value = MakeField("value", arrow::int32(), 2);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key0, key1, value});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key0, key1, value});
-    std::shared_ptr<arrow::Schema> actual_schema = MakePreparedSchema({key0, value});
-    std::shared_ptr<arrow::DataType> actual_type = arrow::struct_(actual_schema->fields());
-    std::shared_ptr<arrow::Array> actual =
-        arrow::ipc::internal::json::ArrayFromJSON(actual_type, R"([[0, 10, 0, 1, 20]])")
-            .ValueOrDie();
-
-    auto batch_reader = std::make_unique<MockFileBatchReader>(actual, actual_type, 1);
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
-                         CreatePreparedQueryReaderForTest(
-                             std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
-                             arrow::schema({key0, key1}), value_schema, pool_));
-    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "field count");
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestQueryReaderRequiresStoreAlignedSchema) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Field> old_value = MakeField("old_value", arrow::int32(), 1);
-    std::shared_ptr<arrow::Field> renamed_value = MakeField("renamed_value", arrow::int32(), 1);
-    std::shared_ptr<arrow::Field> added = MakeField("added", arrow::int32(), 2);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key, renamed_value, added});
-    std::shared_ptr<arrow::Schema> prepared_schema =
-        MakePreparedSchema({key, renamed_value, added});
-    std::shared_ptr<arrow::Schema> actual_schema = MakePreparedSchema({key, old_value});
-    std::shared_ptr<arrow::DataType> actual_type = arrow::struct_(actual_schema->fields());
-    std::shared_ptr<arrow::Array> actual =
-        arrow::ipc::internal::json::ArrayFromJSON(actual_type, R"([[0, 10, 0, 1, 20]])")
-            .ValueOrDie();
-
-    auto batch_reader = std::make_unique<MockFileBatchReader>(actual, actual_type, 1);
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
-                         CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
-                                                          OffsetRange(0, 1), arrow::schema({key}),
-                                                          value_schema, pool_));
-    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "field count");
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderSafeDecode) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-
-    arrow::FieldVector invalid_fields = prepared_schema->fields();
-    invalid_fields[0] = invalid_fields[0]->WithName("wrong_value_kind");
-    invalid_fields[3] = MakeField("wrong_key", arrow::int32(), 99);
-    std::shared_ptr<arrow::DataType> invalid_type = arrow::struct_(invalid_fields);
-    auto invalid_array = std::dynamic_pointer_cast<arrow::StructArray>(
-        arrow::ipc::internal::json::ArrayFromJSON(invalid_type, R"([[0, 10, 0, 1]])").ValueOrDie());
-
-    auto batch_reader = std::make_unique<MockFileBatchReader>(invalid_array, invalid_type, 1);
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
-                                         OffsetRange(0, 1), value_schema, value_schema, pool_));
-    ASSERT_NOK_WITH_MSG(
-        (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
-                                                    KeyValueRecordReader::Iterator>(reader.get())),
-        "prepared batch field");
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderNestedValues) {
-    std::shared_ptr<arrow::Field> id = MakeField("id", arrow::int32(), 0);
-    std::shared_ptr<arrow::Schema> key_schema = arrow::schema({id});
-    std::shared_ptr<arrow::Field> query_item_b = MakeField("renamed_b", arrow::int32(), 11);
-    std::shared_ptr<arrow::Field> query_item_a = MakeField("renamed_a", arrow::int32(), 10);
-    std::shared_ptr<arrow::Field> query_items = MakeField(
-        "items_renamed",
-        arrow::list(arrow::field("element", arrow::struct_({query_item_b, query_item_a}))), 2);
-    std::shared_ptr<arrow::Field> query_attr_y = MakeField("renamed_y", arrow::int32(), 21);
-    std::shared_ptr<arrow::Field> query_attr_x = MakeField("renamed_x", arrow::int32(), 20);
-    std::shared_ptr<arrow::Field> query_attrs =
-        MakeField("attrs_renamed",
-                  arrow::map(arrow::utf8(), arrow::struct_({query_attr_y, query_attr_x})), 3);
-    std::shared_ptr<arrow::Field> query_key_right = MakeField("renamed_right", arrow::int32(), 31);
-    std::shared_ptr<arrow::Field> query_key_left = MakeField("renamed_left", arrow::int32(), 30);
-    std::shared_ptr<arrow::Field> query_keyed_values =
-        MakeField("keyed_values_renamed",
-                  arrow::map(arrow::struct_({query_key_right, query_key_left}), arrow::int32()), 4);
-    std::shared_ptr<arrow::Schema> query_value_schema =
-        arrow::schema({id, query_items, query_attrs, query_keyed_values});
-    std::shared_ptr<arrow::Schema> prepared_schema =
-        MakePreparedSchema(query_value_schema->fields());
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
-        arrow::ipc::internal::json::ArrayFromJSON(
-            prepared_type,
-            R"([[0, 10, 0, 1, [[200, 100], [400, 300]], [["k1", [8, 7]], ["k2", [10, 9]]], [[[12, 11], 13], [[22, 21], 23]]]])")
-            .ValueOrDie();
-
-    auto batch_reader = std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
-                                         OffsetRange(0, 1), key_schema, query_value_schema, pool_));
-    ASSERT_OK_AND_ASSIGN(
-        std::vector<KeyValue> results,
-        (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
-                                                    KeyValueRecordReader::Iterator>(reader.get())));
-
-    ASSERT_EQ(results.size(), 1);
-    ASSERT_EQ(results[0].key->GetInt(0), 1);
-    ASSERT_EQ(results[0].value->GetFieldCount(), 4);
-    ASSERT_EQ(results[0].value->GetInt(0), 1);
-
-    std::shared_ptr<InternalArray> item_array = results[0].value->GetArray(1);
-    ASSERT_EQ(item_array->Size(), 2);
-    std::shared_ptr<InternalRow> first_item = item_array->GetRow(0, 2);
-    ASSERT_EQ(first_item->GetInt(0), 200);
-    ASSERT_EQ(first_item->GetInt(1), 100);
-    std::shared_ptr<InternalRow> second_item = item_array->GetRow(1, 2);
-    ASSERT_EQ(second_item->GetInt(0), 400);
-    ASSERT_EQ(second_item->GetInt(1), 300);
-
-    std::shared_ptr<InternalMap> attr_map = results[0].value->GetMap(2);
-    ASSERT_EQ(attr_map->Size(), 2);
-    std::shared_ptr<InternalArray> key_array = attr_map->KeyArray();
-    ASSERT_EQ(std::string(key_array->GetStringView(0)), "k1");
-    ASSERT_EQ(std::string(key_array->GetStringView(1)), "k2");
-    std::shared_ptr<InternalArray> value_array = attr_map->ValueArray();
-    std::shared_ptr<InternalRow> first_attr = value_array->GetRow(0, 2);
-    ASSERT_EQ(first_attr->GetInt(0), 8);
-    ASSERT_EQ(first_attr->GetInt(1), 7);
-    std::shared_ptr<InternalRow> second_attr = value_array->GetRow(1, 2);
-    ASSERT_EQ(second_attr->GetInt(0), 10);
-    ASSERT_EQ(second_attr->GetInt(1), 9);
-
-    std::shared_ptr<InternalMap> keyed_value_map = results[0].value->GetMap(3);
-    ASSERT_EQ(keyed_value_map->Size(), 2);
-    std::shared_ptr<InternalArray> struct_keys = keyed_value_map->KeyArray();
-    std::shared_ptr<InternalRow> first_key = struct_keys->GetRow(0, 2);
-    ASSERT_EQ(first_key->GetInt(0), 12);
-    ASSERT_EQ(first_key->GetInt(1), 11);
-    std::shared_ptr<InternalRow> second_key = struct_keys->GetRow(1, 2);
-    ASSERT_EQ(second_key->GetInt(0), 22);
-    ASSERT_EQ(second_key->GetInt(1), 21);
-    ASSERT_EQ(keyed_value_map->ValueArray()->GetInt(0), 13);
-    ASSERT_EQ(keyed_value_map->ValueArray()->GetInt(1), 23);
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderFactoryFailureClosesReaders) {
-    std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32())),
-                                           DataField(1, arrow::field("v0", arrow::int32()))};
-    std::shared_ptr<arrow::Schema> value_schema =
-        DataField::ConvertDataFieldsToArrowSchema(value_fields);
-    std::shared_ptr<arrow::Schema> key_schema = arrow::schema({value_schema->field(0)});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema(value_schema->fields());
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
-        [0, 10, 0, 1, 100]
-    ])")
-            .ValueOrDie());
-
-    int32_t factory_failure_close_count = 0;
-    std::vector<std::unique_ptr<BatchReader>> batch_readers;
-    batch_readers.push_back(std::make_unique<TrackingBatchReader>(
-        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1),
-        &factory_failure_close_count));
-    batch_readers.push_back(nullptr);
-    ASSERT_NOK_WITH_MSG(PreparedKeyValueReaderFactory::CreateForQuery(
-                            std::move(batch_readers), prepared_schema, OffsetRange(0, 1),
-                            key_schema, value_schema, pool_),
-                        "PK real-time store returned a null query reader");
-    ASSERT_EQ(factory_failure_close_count, 1);
-}
-
 }  // namespace paimon::test
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index c2d729900..d5e29da52 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -169,38 +169,6 @@ Result<std::vector<int32_t>> ResolveFieldIndexes(
     return result;
 }
 
-Status ValidateReaderParameters(const std::shared_ptr<arrow::Schema>& prepared_schema,
-                                const std::shared_ptr<arrow::Schema>& key_schema,
-                                const std::shared_ptr<arrow::Schema>& value_schema,
-                                const std::shared_ptr<MemoryPool>& memory_pool) {
-    PAIMON_RETURN_NOT_OK(PreparedKeyValueReaderFactory::ValidateTransportSchema(prepared_schema));
-    if (!key_schema) {
-        return Status::Invalid("prepared key schema cannot be null");
-    }
-    if (!value_schema) {
-        return Status::Invalid("prepared value schema cannot be null");
-    }
-    if (!memory_pool) {
-        return Status::Invalid("prepared reader memory pool cannot be null");
-    }
-    return Status::OK();
-}
-
-Status ValidateExactCommitSchema(const std::shared_ptr<arrow::Schema>& prepared_schema,
-                                 const std::shared_ptr<arrow::Schema>& value_schema) {
-    if (prepared_schema->num_fields() !=
-        value_schema->num_fields() + SpecialFields::kPreparedKeyValueValueStartIndex) {
-        return Status::Invalid("commit requires the exact prepared writer schema");
-    }
-    for (int32_t i = 0; i < value_schema->num_fields(); ++i) {
-        if (!prepared_schema->field(i + SpecialFields::kPreparedKeyValueValueStartIndex)
-                 ->Equals(value_schema->field(i), true)) {
-            return Status::Invalid("commit requires the exact prepared writer schema");
-        }
-    }
-    return Status::OK();
-}
-
 class PreparedReaderPlan {
  public:
     static Result<std::shared_ptr<const PreparedReaderPlan>> Create(
@@ -471,19 +439,6 @@ Status PreparedKeyValueReaderFactory::ValidateTransportSchema(
     return Status::OK();
 }
 
-namespace {
-
-std::unique_ptr<KeyValueRecordReader> AdaptPreparedBatchReader(
-    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<const PreparedReaderPlan>& plan,
-    const std::optional<OffsetRange>& visible_offsets,
-    const std::shared_ptr<MemoryPool>& memory_pool,
-    const std::shared_ptr<RealtimeOffsetCoverage>& offset_coverage) {
-    return std::make_unique<PreparedKeyValueReader>(std::move(reader), plan, visible_offsets,
-                                                    memory_pool, offset_coverage);
-}
-
-}  // namespace
-
 Result<std::vector<std::unique_ptr<KeyValueRecordReader>>>
 PreparedKeyValueReaderFactory::CreateForQuery(std::vector<std::unique_ptr<BatchReader>>&& readers,
                                               const std::shared_ptr<arrow::Schema>& prepared_schema,
@@ -493,9 +448,6 @@ PreparedKeyValueReaderFactory::CreateForQuery(std::vector<std::unique_ptr<BatchR
                                               const std::shared_ptr<MemoryPool>& memory_pool) {
     std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers;
     ScopeGuard remaining_raw_readers_guard([&readers]() { CloseReaders(readers); });
-    if (visible_offsets.begin > visible_offsets.end) {
-        return Status::Invalid("prepared visible offset range begin exceeds end");
-    }
     if (readers.empty() && visible_offsets.begin < visible_offsets.end) {
         return Status::Invalid(
             "PK real-time store returned no query readers for a non-empty visible range");
@@ -505,8 +457,7 @@ PreparedKeyValueReaderFactory::CreateForQuery(std::vector<std::unique_ptr<BatchR
             return Status::Invalid("PK real-time store returned a null query reader");
         }
     }
-    PAIMON_RETURN_NOT_OK(
-        ValidateReaderParameters(prepared_schema, key_schema, value_schema, memory_pool));
+    PAIMON_RETURN_NOT_OK(ValidateTransportSchema(prepared_schema));
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<const PreparedReaderPlan> plan,
                            PreparedReaderPlan::Create(prepared_schema, key_schema, value_schema));
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeOffsetCoverage> offset_coverage,
@@ -514,8 +465,8 @@ PreparedKeyValueReaderFactory::CreateForQuery(std::vector<std::unique_ptr<BatchR
                                                           /*allow_committed_prefix=*/true));
     adapted_readers.reserve(readers.size());
     for (std::unique_ptr<BatchReader>& reader : readers) {
-        adapted_readers.push_back(AdaptPreparedBatchReader(std::move(reader), plan, visible_offsets,
-                                                           memory_pool, offset_coverage));
+        adapted_readers.push_back(std::make_unique<PreparedKeyValueReader>(
+            std::move(reader), plan, visible_offsets, memory_pool, offset_coverage));
     }
     remaining_raw_readers_guard.Release();
     return adapted_readers;
@@ -539,9 +490,7 @@ PreparedKeyValueReaderFactory::CreateForCommit(
             return Status::Invalid("PK real-time store returned a null commit reader");
         }
     }
-    PAIMON_RETURN_NOT_OK(
-        ValidateReaderParameters(prepared_schema, key_schema, value_schema, memory_pool));
-    PAIMON_RETURN_NOT_OK(ValidateExactCommitSchema(prepared_schema, value_schema));
+    PAIMON_RETURN_NOT_OK(ValidateTransportSchema(prepared_schema));
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<const PreparedReaderPlan> plan,
                            PreparedReaderPlan::Create(prepared_schema, key_schema, value_schema));
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeOffsetCoverage> offset_coverage,
@@ -549,8 +498,8 @@ PreparedKeyValueReaderFactory::CreateForCommit(
                                                           /*allow_committed_prefix=*/false));
     adapted_readers.reserve(readers.size());
     for (std::unique_ptr<BatchReader>& reader : readers) {
-        adapted_readers.push_back(AdaptPreparedBatchReader(std::move(reader), plan, std::nullopt,
-                                                           memory_pool, offset_coverage));
+        adapted_readers.push_back(std::make_unique<PreparedKeyValueReader>(
+            std::move(reader), plan, std::nullopt, memory_pool, offset_coverage));
     }
     remaining_raw_readers_guard.Release();
     return adapted_readers;
diff --git a/src/paimon/core/realtime/prepared_key_value_reader_test.cpp b/src/paimon/core/realtime/prepared_key_value_reader_test.cpp
new file mode 100644
index 000000000..c0515df72
--- /dev/null
+++ b/src/paimon/core/realtime/prepared_key_value_reader_test.cpp
@@ -0,0 +1,621 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/realtime/prepared_key_value_reader.h"
+
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "arrow/api.h"
+#include "arrow/array/array_nested.h"
+#include "arrow/ipc/json_simple.h"
+#include "gtest/gtest.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/realtime/offset_range.h"
+#include "paimon/testing/mock/mock_file_batch_reader.h"
+#include "paimon/testing/utils/key_value_checker.h"
+#include "paimon/testing/utils/read_result_collector.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+namespace {
+
+std::shared_ptr<arrow::Field> MakeField(const std::string& name,
+                                        const std::shared_ptr<arrow::DataType>& type,
+                                        int32_t field_id, bool nullable = true) {
+    return DataField::ConvertDataFieldToArrowField(
+        DataField(field_id, arrow::field(name, type, nullable)));
+}
+
+std::shared_ptr<arrow::Schema> MakePreparedSchema(const arrow::FieldVector& value_fields) {
+    arrow::FieldVector prepared_fields = {
+        DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
+        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
+            ->WithNullable(false),
+        DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset())};
+    prepared_fields.insert(prepared_fields.end(), value_fields.begin(), value_fields.end());
+    return arrow::schema(prepared_fields);
+}
+
+Result<std::unique_ptr<KeyValueRecordReader>> CreatePreparedQueryReaderForTest(
+    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const OffsetRange& visible_offsets, const std::shared_ptr<arrow::Schema>& key_schema,
+    const std::shared_ptr<arrow::Schema>& value_schema,
+    const std::shared_ptr<MemoryPool>& memory_pool) {
+    std::vector<std::unique_ptr<BatchReader>> readers;
+    readers.push_back(std::move(reader));
+    PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers,
+                           PreparedKeyValueReaderFactory::CreateForQuery(
+                               std::move(readers), prepared_schema, visible_offsets, key_schema,
+                               value_schema, memory_pool));
+    return std::move(adapted_readers[0]);
+}
+
+Result<std::unique_ptr<KeyValueRecordReader>> CreatePreparedCommitReaderForTest(
+    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const OffsetRange& sealed_offsets, const std::shared_ptr<arrow::Schema>& key_schema,
+    const std::shared_ptr<arrow::Schema>& value_schema,
+    const std::shared_ptr<MemoryPool>& memory_pool) {
+    std::vector<std::unique_ptr<BatchReader>> readers;
+    readers.push_back(std::move(reader));
+    PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers,
+                           PreparedKeyValueReaderFactory::CreateForCommit(
+                               std::move(readers), prepared_schema, sealed_offsets, key_schema,
+                               value_schema, memory_pool));
+    return std::move(adapted_readers[0]);
+}
+
+class TrackingBatchReader : public BatchReader {
+ public:
+    TrackingBatchReader(std::unique_ptr<BatchReader>&& delegate, int32_t* close_count)
+        : delegate_(std::move(delegate)), close_count_(close_count) {}
+
+    Result<ReadBatch> NextBatch() override {
+        return delegate_->NextBatch();
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return delegate_->GetReaderMetrics();
+    }
+
+    void Close() override {
+        ++(*close_count_);
+        delegate_->Close();
+    }
+
+ private:
+    std::unique_ptr<BatchReader> delegate_;
+    int32_t* close_count_;
+};
+
+class MalformedBitmapBatchReader : public BatchReader {
+ public:
+    MalformedBitmapBatchReader(std::unique_ptr<BatchReader>&& delegate, int32_t row_id)
+        : delegate_(std::move(delegate)), row_id_(row_id) {}
+
+    Result<ReadBatch> NextBatch() override {
+        return delegate_->NextBatch();
+    }
+
+    Result<ReadBatchWithBitmap> NextBatchWithBitmap() override {
+        PAIMON_ASSIGN_OR_RAISE(ReadBatchWithBitmap batch, delegate_->NextBatchWithBitmap());
+        if (!IsEofBatch(batch)) {
+            batch.second.Add(row_id_);
+        }
+        return batch;
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return delegate_->GetReaderMetrics();
+    }
+
+    void Close() override {
+        delegate_->Close();
+    }
+
+ private:
+    std::unique_ptr<BatchReader> delegate_;
+    int32_t row_id_;
+};
+
+}  // namespace
+
+class PreparedKeyValueReaderTest : public testing::Test {
+ protected:
+    std::shared_ptr<MemoryPool> pool_ = GetDefaultPool();
+};
+
+TEST_F(PreparedKeyValueReaderTest, TestQueryAllowsCommittedPrefix) {
+    std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32())),
+                                           DataField(1, arrow::field("v0", arrow::int32()))};
+    std::shared_ptr<arrow::Schema> value_schema =
+        DataField::ConvertDataFieldsToArrowSchema(value_fields);
+    std::shared_ptr<arrow::Schema> key_schema = arrow::schema({value_schema->field(0)});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema(value_schema->fields());
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
+        [0, 100, 0, 1, 10],
+        [0, 101, 1, 2, 20],
+        [0, 102, 2, 4, 40],
+        [0, 103, 3, 6, 60]
+    ])")
+            .ValueOrDie());
+
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+    batch_readers.push_back(
+        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 2));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
+                         PreparedKeyValueReaderFactory::CreateForQuery(
+                             std::move(batch_readers), prepared_schema, OffsetRange(2, 4),
+                             key_schema, value_schema, pool_));
+    ASSERT_EQ(1, readers.size());
+    ASSERT_OK_AND_ASSIGN(
+        std::vector<KeyValue> results,
+        (ReadResultCollector::CollectKeyValueResult<
+            KeyValueRecordReader, KeyValueRecordReader::Iterator>(readers[0].get())));
+
+    std::vector<RowKind*> row_kinds = {const_cast<RowKind*>(RowKind::Insert()),
+                                       const_cast<RowKind*>(RowKind::Insert())};
+    std::vector<int64_t> levels = {KeyValue::UNKNOWN_LEVEL, KeyValue::UNKNOWN_LEVEL};
+    std::vector<KeyValue> expected = KeyValueChecker::GenerateKeyValues(
+        row_kinds, {102, 103}, levels, {{4}, {6}}, {{4, 40}, {6, 60}}, pool_);
+    KeyValueChecker::CheckResult(expected, results, 1, 2);
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestQueryRejectsNegativeOffset) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, -1, 1]])")
+            .ValueOrDie();
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        CreatePreparedQueryReaderForTest(
+            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type,
+                                                  /*read_batch_size=*/1),
+            prepared_schema, OffsetRange(1, 2), value_schema, value_schema, pool_));
+
+    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "reader offset must be non-negative");
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestQueryOffsetCoverageAcrossReadersAndBatches) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> first_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+                                                  R"([[0, 10, 2, 1], [0, 11, 0, 2]])")
+            .ValueOrDie();
+    std::shared_ptr<arrow::Array> second_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+                                                  R"([[0, 12, 3, 3], [0, 13, 1, 4]])")
+            .ValueOrDie();
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+    batch_readers.push_back(
+        std::make_unique<MockFileBatchReader>(first_array, prepared_type, /*read_batch_size=*/1));
+    batch_readers.push_back(
+        std::make_unique<MockFileBatchReader>(second_array, prepared_type, /*read_batch_size=*/1));
+
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
+                         PreparedKeyValueReaderFactory::CreateForQuery(
+                             std::move(batch_readers), prepared_schema, OffsetRange(0, 4),
+                             value_schema, value_schema, pool_));
+    int64_t row_count = 0;
+    for (const std::unique_ptr<KeyValueRecordReader>& reader : readers) {
+        ASSERT_OK_AND_ASSIGN(
+            std::vector<KeyValue> rows,
+            (ReadResultCollector::CollectKeyValueResult<
+                KeyValueRecordReader, KeyValueRecordReader::Iterator>(reader.get())));
+        row_count += static_cast<int64_t>(rows.size());
+    }
+    ASSERT_EQ(4, row_count);
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestQueryRejectsMissingVisibleOffset) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+                                                  R"([[0, 10, 0, 1], [0, 11, 2, 2]])")
+            .ValueOrDie();
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        CreatePreparedQueryReaderForTest(
+            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type,
+                                                  /*read_batch_size=*/1),
+            prepared_schema, OffsetRange(0, 3), value_schema, value_schema, pool_));
+
+    ASSERT_NOK_WITH_MSG(
+        (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
+                                                    KeyValueRecordReader::Iterator>(reader.get())),
+        "query readers did not cover the visible range");
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestQueryRejectsDuplicateVisibleOffset) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> first_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1]])").ValueOrDie();
+    std::shared_ptr<arrow::Array> second_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+                                                  R"([[0, 11, 1, 2], [0, 12, 1, 3]])")
+            .ValueOrDie();
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+    batch_readers.push_back(
+        std::make_unique<MockFileBatchReader>(first_array, prepared_type, /*read_batch_size=*/1));
+    batch_readers.push_back(
+        std::make_unique<MockFileBatchReader>(second_array, prepared_type, /*read_batch_size=*/1));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
+                         PreparedKeyValueReaderFactory::CreateForQuery(
+                             std::move(batch_readers), prepared_schema, OffsetRange(0, 2),
+                             value_schema, value_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(
+        std::vector<KeyValue> first_rows,
+        (ReadResultCollector::CollectKeyValueResult<
+            KeyValueRecordReader, KeyValueRecordReader::Iterator>(readers[0].get())));
+    ASSERT_EQ(1, first_rows.size());
+    ASSERT_NOK_WITH_MSG((ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
+                                                                    KeyValueRecordReader::Iterator>(
+                            readers[1].get())),
+                        "query readers did not cover the visible range");
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestQueryRejectsEmptyEofForVisibleRange) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([])").ValueOrDie();
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        CreatePreparedQueryReaderForTest(
+            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type,
+                                                  /*read_batch_size=*/1),
+            prepared_schema, OffsetRange(0, 1), value_schema, value_schema, pool_));
+
+    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "query readers did not cover the visible range");
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestQueryRejectsEmptyReadersForVisibleRange) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+
+    ASSERT_NOK_WITH_MSG(
+        PreparedKeyValueReaderFactory::CreateForQuery(std::move(batch_readers), prepared_schema,
+                                                      OffsetRange(0, 1), value_schema, value_schema,
+                                                      pool_),
+        "PK real-time store returned no query readers for a non-empty visible range");
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestQueryAllowsEmptyReadersForEmptyVisibleRange) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
+                         PreparedKeyValueReaderFactory::CreateForQuery(
+                             std::move(batch_readers), prepared_schema, OffsetRange(1, 1),
+                             value_schema, value_schema, pool_));
+    ASSERT_TRUE(readers.empty());
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestPreparedReaderBitmapBounds) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1]])").ValueOrDie();
+    auto batch_reader = std::make_unique<MalformedBitmapBatchReader>(
+        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, /*batch_size=*/1),
+        /*row_id=*/1);
+
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
+                                         OffsetRange(0, 1), value_schema, value_schema, pool_));
+    Result<std::vector<KeyValue>> result =
+        ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
+                                                   KeyValueRecordReader::Iterator>(reader.get());
+    ASSERT_TRUE(result.status().IsInvalid());
+    ASSERT_NOK_WITH_MSG(result, "selected row id 1 is out of bounds for prepared batch length 1");
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestPreparedQueryReaderRejectsPartialBitmap) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+                                                  R"([[0, 10, 0, 1], [0, 11, 1, 2]])")
+            .ValueOrDie();
+    RoaringBitmap32 partial_bitmap;
+    partial_bitmap.Add(0);
+    auto batch_reader = std::make_unique<MockFileBatchReader>(
+        prepared_array, prepared_type, partial_bitmap, /*read_batch_size=*/2);
+    batch_reader->EnableRandomizeBatchSize(false);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
+                                         OffsetRange(0, 2), value_schema, value_schema, pool_));
+
+    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "must cover every raw mutation");
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestPreparedReaderQueryProjection) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Field> extra = MakeField("extra", arrow::int32(), 1);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key, extra});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1, 2]])")
+            .ValueOrDie());
+
+    auto query_batch_reader =
+        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> query_reader,
+        CreatePreparedQueryReaderForTest(std::move(query_batch_reader), prepared_schema,
+                                         OffsetRange(0, 1), value_schema, value_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(
+        std::vector<KeyValue> query_results,
+        (ReadResultCollector::CollectKeyValueResult<
+            KeyValueRecordReader, KeyValueRecordReader::Iterator>(query_reader.get())));
+    ASSERT_EQ(query_results.size(), 1);
+    ASSERT_EQ(query_results[0].value->GetFieldCount(), 1);
+    ASSERT_EQ(query_results[0].value->GetInt(0), 1);
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestCommitOffsetCoverage) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> first_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+                                                  R"([[0, 10, 2, 1], [0, 11, 0, 3]])")
+            .ValueOrDie();
+    std::shared_ptr<arrow::Array> second_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+                                                  R"([[0, 12, 1, 2], [0, 13, 3, 4]])")
+            .ValueOrDie();
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+    batch_readers.push_back(
+        std::make_unique<MockFileBatchReader>(first_array, prepared_type, /*read_batch_size=*/1));
+    batch_readers.push_back(
+        std::make_unique<MockFileBatchReader>(second_array, prepared_type, /*read_batch_size=*/1));
+
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
+                         PreparedKeyValueReaderFactory::CreateForCommit(
+                             std::move(batch_readers), prepared_schema, OffsetRange(0, 4),
+                             value_schema, value_schema, pool_));
+    int64_t row_count = 0;
+    for (const std::unique_ptr<KeyValueRecordReader>& reader : readers) {
+        ASSERT_OK_AND_ASSIGN(
+            std::vector<KeyValue> rows,
+            (ReadResultCollector::CollectKeyValueResult<
+                KeyValueRecordReader, KeyValueRecordReader::Iterator>(reader.get())));
+        row_count += static_cast<int64_t>(rows.size());
+    }
+    ASSERT_EQ(4, row_count);
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestCommitRejectsEmptyReaders) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+
+    ASSERT_NOK_WITH_MSG(PreparedKeyValueReaderFactory::CreateForCommit(
+                            std::move(batch_readers), prepared_schema, OffsetRange(0, 1),
+                            value_schema, value_schema, pool_),
+                        "PK real-time store returned no commit readers for a sealed segment");
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestRejectsDuplicateCommitOffset) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(
+            prepared_type, R"([[0, 10, 0, 1], [0, 11, 0, 2], [0, 12, 2, 3]])")
+            .ValueOrDie();
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+    batch_readers.push_back(std::make_unique<MockFileBatchReader>(prepared_array, prepared_type,
+                                                                  /*read_batch_size=*/1));
+
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
+                         PreparedKeyValueReaderFactory::CreateForCommit(
+                             std::move(batch_readers), prepared_schema, OffsetRange(0, 3),
+                             value_schema, value_schema, pool_));
+    ASSERT_NOK_WITH_MSG((ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
+                                                                    KeyValueRecordReader::Iterator>(
+                            readers[0].get())),
+                        "did not cover the sealed range");
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestBadCommitBatch) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Field> value = MakeField("value", arrow::int32(), 1);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key, value});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key, value});
+    std::shared_ptr<arrow::Schema> actual_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> actual_type = arrow::struct_(actual_schema->fields());
+    std::shared_ptr<arrow::Array> actual =
+        arrow::ipc::internal::json::ArrayFromJSON(actual_type, R"([[0, 10, 0, 1]])").ValueOrDie();
+
+    auto batch_reader = std::make_unique<MockFileBatchReader>(actual, actual_type, 1);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
+                         CreatePreparedCommitReaderForTest(std::move(batch_reader), prepared_schema,
+                                                           OffsetRange(0, 1), arrow::schema({key}),
+                                                           value_schema, pool_));
+    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "field count");
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestPreparedReaderSafeDecode) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+
+    arrow::FieldVector invalid_fields = prepared_schema->fields();
+    invalid_fields[0] = invalid_fields[0]->WithName("wrong_value_kind");
+    invalid_fields[3] = MakeField("wrong_key", arrow::int32(), 99);
+    std::shared_ptr<arrow::DataType> invalid_type = arrow::struct_(invalid_fields);
+    auto invalid_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(invalid_type, R"([[0, 10, 0, 1]])").ValueOrDie());
+
+    auto batch_reader = std::make_unique<MockFileBatchReader>(invalid_array, invalid_type, 1);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
+                                         OffsetRange(0, 1), value_schema, value_schema, pool_));
+    ASSERT_NOK_WITH_MSG(
+        (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
+                                                    KeyValueRecordReader::Iterator>(reader.get())),
+        "prepared batch field");
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestPreparedReaderNestedValues) {
+    std::shared_ptr<arrow::Field> id = MakeField("id", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> key_schema = arrow::schema({id});
+    std::shared_ptr<arrow::Field> query_item_b = MakeField("renamed_b", arrow::int32(), 11);
+    std::shared_ptr<arrow::Field> query_item_a = MakeField("renamed_a", arrow::int32(), 10);
+    std::shared_ptr<arrow::Field> query_items = MakeField(
+        "items_renamed",
+        arrow::list(arrow::field("element", arrow::struct_({query_item_b, query_item_a}))), 2);
+    std::shared_ptr<arrow::Field> query_attr_y = MakeField("renamed_y", arrow::int32(), 21);
+    std::shared_ptr<arrow::Field> query_attr_x = MakeField("renamed_x", arrow::int32(), 20);
+    std::shared_ptr<arrow::Field> query_attrs =
+        MakeField("attrs_renamed",
+                  arrow::map(arrow::utf8(), arrow::struct_({query_attr_y, query_attr_x})), 3);
+    std::shared_ptr<arrow::Field> query_key_right = MakeField("renamed_right", arrow::int32(), 31);
+    std::shared_ptr<arrow::Field> query_key_left = MakeField("renamed_left", arrow::int32(), 30);
+    std::shared_ptr<arrow::Field> query_keyed_values =
+        MakeField("keyed_values_renamed",
+                  arrow::map(arrow::struct_({query_key_right, query_key_left}), arrow::int32()), 4);
+    std::shared_ptr<arrow::Schema> query_value_schema =
+        arrow::schema({id, query_items, query_attrs, query_keyed_values});
+    std::shared_ptr<arrow::Schema> prepared_schema =
+        MakePreparedSchema(query_value_schema->fields());
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(
+            prepared_type,
+            R"([[0, 10, 0, 1, [[200, 100], [400, 300]], [["k1", [8, 7]], ["k2", [10, 9]]], [[[12, 11], 13], [[22, 21], 23]]]])")
+            .ValueOrDie();
+
+    auto batch_reader = std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
+                                         OffsetRange(0, 1), key_schema, query_value_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(
+        std::vector<KeyValue> results,
+        (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
+                                                    KeyValueRecordReader::Iterator>(reader.get())));
+
+    ASSERT_EQ(results.size(), 1);
+    ASSERT_EQ(results[0].key->GetInt(0), 1);
+    ASSERT_EQ(results[0].value->GetFieldCount(), 4);
+    ASSERT_EQ(results[0].value->GetInt(0), 1);
+
+    std::shared_ptr<InternalArray> item_array = results[0].value->GetArray(1);
+    ASSERT_EQ(item_array->Size(), 2);
+    std::shared_ptr<InternalRow> first_item = item_array->GetRow(0, 2);
+    ASSERT_EQ(first_item->GetInt(0), 200);
+    ASSERT_EQ(first_item->GetInt(1), 100);
+    std::shared_ptr<InternalRow> second_item = item_array->GetRow(1, 2);
+    ASSERT_EQ(second_item->GetInt(0), 400);
+    ASSERT_EQ(second_item->GetInt(1), 300);
+
+    std::shared_ptr<InternalMap> attr_map = results[0].value->GetMap(2);
+    ASSERT_EQ(attr_map->Size(), 2);
+    std::shared_ptr<InternalArray> key_array = attr_map->KeyArray();
+    ASSERT_EQ(std::string(key_array->GetStringView(0)), "k1");
+    ASSERT_EQ(std::string(key_array->GetStringView(1)), "k2");
+    std::shared_ptr<InternalArray> value_array = attr_map->ValueArray();
+    std::shared_ptr<InternalRow> first_attr = value_array->GetRow(0, 2);
+    ASSERT_EQ(first_attr->GetInt(0), 8);
+    ASSERT_EQ(first_attr->GetInt(1), 7);
+    std::shared_ptr<InternalRow> second_attr = value_array->GetRow(1, 2);
+    ASSERT_EQ(second_attr->GetInt(0), 10);
+    ASSERT_EQ(second_attr->GetInt(1), 9);
+
+    std::shared_ptr<InternalMap> keyed_value_map = results[0].value->GetMap(3);
+    ASSERT_EQ(keyed_value_map->Size(), 2);
+    std::shared_ptr<InternalArray> struct_keys = keyed_value_map->KeyArray();
+    std::shared_ptr<InternalRow> first_key = struct_keys->GetRow(0, 2);
+    ASSERT_EQ(first_key->GetInt(0), 12);
+    ASSERT_EQ(first_key->GetInt(1), 11);
+    std::shared_ptr<InternalRow> second_key = struct_keys->GetRow(1, 2);
+    ASSERT_EQ(second_key->GetInt(0), 22);
+    ASSERT_EQ(second_key->GetInt(1), 21);
+    ASSERT_EQ(keyed_value_map->ValueArray()->GetInt(0), 13);
+    ASSERT_EQ(keyed_value_map->ValueArray()->GetInt(1), 23);
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestPreparedReaderFactoryFailureClosesReaders) {
+    std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32())),
+                                           DataField(1, arrow::field("v0", arrow::int32()))};
+    std::shared_ptr<arrow::Schema> value_schema =
+        DataField::ConvertDataFieldsToArrowSchema(value_fields);
+    std::shared_ptr<arrow::Schema> key_schema = arrow::schema({value_schema->field(0)});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema(value_schema->fields());
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
+        [0, 10, 0, 1, 100]
+    ])")
+            .ValueOrDie());
+
+    int32_t factory_failure_close_count = 0;
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+    batch_readers.push_back(std::make_unique<TrackingBatchReader>(
+        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1),
+        &factory_failure_close_count));
+    batch_readers.push_back(nullptr);
+    ASSERT_NOK_WITH_MSG(PreparedKeyValueReaderFactory::CreateForQuery(
+                            std::move(batch_readers), prepared_schema, OffsetRange(0, 1),
+                            key_schema, value_schema, pool_),
+                        "PK real-time store returned a null query reader");
+    ASSERT_EQ(factory_failure_close_count, 1);
+}
+
+}  // namespace paimon::test

From 87796a390455ba5eff66b23e2ba22fdc270d6b54 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Thu, 27 Aug 2026 21:28:15 +0800
Subject: [PATCH 54/62] fix(read): close realtime readers on setup failure

---
 .../core/mergetree/merge_tree_writer_test.cpp | 28 ------------
 .../core/operation/merge_file_split_read.cpp  | 40 +++++++++++++++--
 .../operation/merge_file_split_read_test.cpp  | 45 +++++++++++++++++--
 3 files changed, 78 insertions(+), 35 deletions(-)

diff --git a/src/paimon/core/mergetree/merge_tree_writer_test.cpp b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
index 179deda44..84ebc0b84 100644
--- a/src/paimon/core/mergetree/merge_tree_writer_test.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
@@ -588,34 +588,6 @@ TEST_P(MergeTreeWriterTest, TestMergeSortedReaders) {
     CheckFileContent(path_factory->ToPath(new_file), expected_array);
 }
 
-TEST_P(MergeTreeWriterTest, TestSortedReaderOwnership) {
-    ASSERT_OK_AND_ASSIGN(CoreOptions options,
-                         CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}}));
-
-    auto dir = UniqueTestDirectory::Create();
-    ASSERT_TRUE(dir);
-    auto path_factory = std::make_shared<DataFilePathFactory>();
-    ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr));
-
-    ASSERT_OK_AND_ASSIGN(auto merge_writer,
-                         CreateMergeWriter(-1, dir->Str(), path_factory, 0, options));
-
-    auto sorted_reader_array = std::dynamic_pointer_cast<arrow::StructArray>(
-        arrow::ipc::internal::json::ArrayFromJSON(write_type_, R"([
-      [0, 0, "Alice", 10, 0, 13.1]
-    ])")
-            .ValueOrDie());
-
-    bool closed = false;
-    std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
-    sorted_readers.push_back(std::make_unique<TrackingKeyValueRecordReader>(
-        CreateSingleReader(sorted_reader_array), &closed));
-
-    ASSERT_OK(merge_writer->WriteSortedReadersToFiles(std::move(sorted_readers)));
-    ASSERT_TRUE(closed);
-    ASSERT_OK(merge_writer->Close());
-}
-
 TEST_P(MergeTreeWriterTest, TestSortedReaderFailure) {
     ASSERT_OK_AND_ASSIGN(CoreOptions options,
                          CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}}));
diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp
index 835ed0932..3b64ed20e 100644
--- a/src/paimon/core/operation/merge_file_split_read.cpp
+++ b/src/paimon/core/operation/merge_file_split_read.cpp
@@ -132,6 +132,13 @@ class MergeFileSplitRead::RealtimeReaderBuilder {
         const std::vector<std::shared_ptr<Split>>& disk_splits,
         std::vector<std::unique_ptr<KeyValueRecordReader>>&& additional_readers,
         MergeFileSplitRead* owner) {
+        ScopeGuard additional_readers_guard([&additional_readers]() {
+            for (const std::unique_ptr<KeyValueRecordReader>& reader : additional_readers) {
+                if (reader) {
+                    reader->Close();
+                }
+            }
+        });
         RealtimeReaderBuilder builder(owner);
         std::vector<std::unique_ptr<KeyValueRecordReader>> readers;
         if (!disk_splits.empty()) {
@@ -141,6 +148,7 @@ class MergeFileSplitRead::RealtimeReaderBuilder {
         for (std::unique_ptr<KeyValueRecordReader>& additional_reader : additional_readers) {
             readers.push_back(std::move(additional_reader));
         }
+        additional_readers_guard.Release();
         return builder.CreateMergedReader(std::move(readers));
     }
 
@@ -219,15 +227,32 @@ class MergeFileSplitRead::RealtimeReaderBuilder {
 
     Result<std::unique_ptr<BatchReader>> CreateMergedReader(
         std::vector<std::unique_ptr<KeyValueRecordReader>>&& record_readers) {
+        ScopeGuard record_readers_guard([&record_readers]() {
+            for (const std::unique_ptr<KeyValueRecordReader>& reader : record_readers) {
+                if (reader) {
+                    reader->Close();
+                }
+            }
+        });
         if (record_readers.empty()) {
+            record_readers_guard.Release();
             return std::make_unique<ConcatBatchReader>(std::vector<std::unique_ptr<BatchReader>>{},
                                                        owner_->pool_);
         }
         PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<SortMergeReader> sort_merge_reader,
                                owner_->CreateSortMergeReader(std::move(record_readers)));
-        return owner_->CreateProjectedReader(std::move(sort_merge_reader),
-                                             owner_->context_->GetPredicate(),
-                                             /*complete_row_kind=*/true);
+        record_readers_guard.Release();
+        ScopeGuard sort_merge_reader_guard([&sort_merge_reader]() {
+            if (sort_merge_reader) {
+                sort_merge_reader->Close();
+            }
+        });
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> result,
+                               owner_->CreateProjectedReader(std::move(sort_merge_reader),
+                                                             owner_->context_->GetPredicate(),
+                                                             /*complete_row_kind=*/true));
+        sort_merge_reader_guard.Release();
+        return result;
     }
 
     MergeFileSplitRead* owner_;
@@ -665,8 +690,15 @@ Result<std::unique_ptr<BatchReader>> MergeFileSplitRead::CreateProjectedReader(
             std::move(sort_merge_reader), raw_read_schema_, projection_,
             options_.GetReadBatchSize(), thread_number, pool_);
     }
-    PAIMON_ASSIGN_OR_RAISE(projection_reader,
+    ScopeGuard projection_reader_guard([&projection_reader]() {
+        if (projection_reader) {
+            projection_reader->Close();
+        }
+    });
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> filtered_reader,
                            ApplyPredicateFilterIfNeeded(std::move(projection_reader), predicate));
+    projection_reader_guard.Release();
+    projection_reader = std::move(filtered_reader);
     if (complete_row_kind) {
         return std::make_unique<CompleteRowKindBatchReader>(std::move(projection_reader), pool_);
     }
diff --git a/src/paimon/core/operation/merge_file_split_read_test.cpp b/src/paimon/core/operation/merge_file_split_read_test.cpp
index 7a859cae5..b79baf5d6 100644
--- a/src/paimon/core/operation/merge_file_split_read_test.cpp
+++ b/src/paimon/core/operation/merge_file_split_read_test.cpp
@@ -71,6 +71,26 @@ namespace {
 
 class TestingSplit : public Split {};
 
+class TrackingKeyValueRecordReader : public KeyValueRecordReader {
+ public:
+    explicit TrackingKeyValueRecordReader(int32_t* close_count) : close_count_(close_count) {}
+
+    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatch() override {
+        return std::unique_ptr<KeyValueRecordReader::Iterator>();
+    }
+
+    void Close() override {
+        ++(*close_count_);
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return nullptr;
+    }
+
+ private:
+    int32_t* close_count_;
+};
+
 }  // namespace
 
 // Parameter: min_heap/loser_tree; enable/disable IO prefetch; enable/disable multi thread row to
@@ -693,7 +713,6 @@ TEST_P(MergeFileSplitReadTest, TestRealtimeReadConcatenatesOrderedDiskSections)
     context_builder.SetOptions(
         {{Options::SEQUENCE_FIELD, "s0,s1"}, {Options::MERGE_ENGINE, "deduplicate"}});
     AddOptions(&context_builder);
-    context_builder.EnableMultiThreadRowToBatch(false);
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<ReadContext> read_context, context_builder.Finish());
     std::shared_ptr<InternalReadContext> internal_context = CreateInternalReadContext(read_context);
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<MergeFileSplitRead> split_read,
@@ -745,14 +764,13 @@ TEST_P(MergeFileSplitReadTest, TestRealtimeReadConcatenatesOrderedDiskSections)
     batch_reader->Close();
 }
 
-TEST_P(MergeFileSplitReadTest, TestRealtimeReadValidatesDiskSplits) {
+TEST_F(MergeFileSplitReadTest, TestRealtimeReadValidatesDiskSplits) {
     std::string path =
         paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor";
     ReadContextBuilder context_builder(path);
     context_builder.SetReadFieldNames({"k0", "k1", "s1", "v0"});
     context_builder.SetOptions(
         {{Options::SEQUENCE_FIELD, "s0,s1"}, {Options::MERGE_ENGINE, "deduplicate"}});
-    AddOptions(&context_builder);
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<ReadContext> read_context, context_builder.Finish());
     std::shared_ptr<InternalReadContext> internal_context = CreateInternalReadContext(read_context);
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<MergeFileSplitRead> split_read,
@@ -778,6 +796,27 @@ TEST_P(MergeFileSplitReadTest, TestRealtimeReadValidatesDiskSplits) {
                         "deletion files must be empty or match data files");
 }
 
+TEST_F(MergeFileSplitReadTest, TestRealtimeReaderFailureClosesPluginReader) {
+    std::string path =
+        paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor";
+    ReadContextBuilder context_builder(path);
+    context_builder.SetReadFieldNames({"k0", "k1", "s1", "v0"});
+    context_builder.SetOptions(
+        {{Options::MERGE_ENGINE, "aggregation"}, {"fields.v0.aggregate-function", "unsupported"}});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<ReadContext> read_context, context_builder.Finish());
+    std::shared_ptr<InternalReadContext> internal_context = CreateInternalReadContext(read_context);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<MergeFileSplitRead> split_read,
+                         CreateMergeFileSplitRead(internal_context));
+
+    int32_t close_count = 0;
+    std::vector<std::unique_ptr<KeyValueRecordReader>> plugin_readers;
+    plugin_readers.push_back(std::make_unique<TrackingKeyValueRecordReader>(&close_count));
+
+    ASSERT_NOK_WITH_MSG(split_read->CreateRealtimeReader({}, std::move(plugin_readers)),
+                        "unsupported");
+    ASSERT_EQ(1, close_count);
+}
+
 TEST_P(MergeFileSplitReadTest, TestLookUp) {
     std::string path =
         paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor";

From 982eb59256ebf14d943e202950b35ad30b8fc671 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Thu, 27 Aug 2026 23:15:44 +0800
Subject: [PATCH 55/62] refactor(realtime): clarify primary-key reader
 contracts

---
 include/paimon/realtime/realtime_store.h      |  23 +-
 src/paimon/CMakeLists.txt                     |   4 +-
 src/paimon/common/table/special_fields.h      |  14 -
 .../common/table/special_fields_test.cpp      |  21 -
 .../merged_key_value_record_reader_test.cpp   |   2 +-
 .../operation/key_value_file_store_write.cpp  |  14 +-
 .../key_value_file_store_write_test.cpp       |  52 ++-
 .../core/operation/merge_file_split_read.cpp  |   1 +
 .../realtime/primary_key_realtime_store.cpp   |  23 +-
 .../realtime/primary_key_realtime_store.h     |   2 +-
 .../primary_key_realtime_store_test.cpp       |  52 +--
 ...er.cpp => realtime_primary_key_reader.cpp} | 234 +++++------
 ...reader.h => realtime_primary_key_reader.h} |  28 +-
 ...p => realtime_primary_key_reader_test.cpp} | 394 ++++++++++--------
 .../realtime/realtime_primary_key_writer.cpp  |  74 ++--
 .../realtime/realtime_primary_key_writer.h    |  13 +-
 .../table/source/key_value_table_read.cpp     |  65 +--
 .../core/table/source/key_value_table_read.h  |   4 +-
 18 files changed, 535 insertions(+), 485 deletions(-)
 rename src/paimon/core/realtime/{prepared_key_value_reader.cpp => realtime_primary_key_reader.cpp} (65%)
 rename src/paimon/core/realtime/{prepared_key_value_reader.h => realtime_primary_key_reader.h} (63%)
 rename src/paimon/core/realtime/{prepared_key_value_reader_test.cpp => realtime_primary_key_reader_test.cpp} (56%)

diff --git a/include/paimon/realtime/realtime_store.h b/include/paimon/realtime/realtime_store.h
index cbfe96595..81e64b9fd 100644
--- a/include/paimon/realtime/realtime_store.h
+++ b/include/paimon/realtime/realtime_store.h
@@ -50,7 +50,7 @@ enum class PAIMON_EXPORT RealtimeStoreMode {
 /// Parameters used by a `RealtimeStoreFactory` to create a store.
 struct PAIMON_EXPORT RealtimeStoreCreateRequest {
     /// Schema whose ownership is transferred to the factory. Append mode receives the complete
-    /// table write schema. Primary-key mode receives the prepared transport schema:
+    /// table write schema. Primary-key mode receives the realtime primary-key transport schema:
     /// [_VALUE_KIND, _SEQUENCE_NUMBER, _REALTIME_OFFSET, table write fields].
     std::unique_ptr<::ArrowSchema> write_schema;
     /// Table options available to the store implementation.
@@ -66,8 +66,8 @@ struct PAIMON_EXPORT RealtimeStoreCreateRequest {
 /// A record batch and its framework-assigned contiguous offset range.
 ///
 /// Append-mode batches contain table write fields, and row `i` has offset
-/// `offset_range.begin + i`. Primary-key batches use the prepared transport schema, are sorted
-/// by full primary key then sequence number, and retain the original offset in
+/// `offset_range.begin + i`. Primary-key batches use the realtime primary-key transport schema,
+/// are sorted by full primary key then sequence number, and retain the original offset in
 /// `_REALTIME_OFFSET`.
 struct PAIMON_EXPORT RealtimeWriteBatch {
     /// Input batch whose ownership is transferred to `RealtimeStore::Write`.
@@ -104,7 +104,8 @@ class PAIMON_EXPORT RealtimeReadView {
 /// Parameters used by a `RealtimeStore` to create readers for a query.
 struct PAIMON_EXPORT RealtimeQueryContext {
     /// Append mode receives the requested output fields before the mandatory leading
-    /// `_VALUE_KIND` field is added. Primary-key mode receives the requested prepared schema.
+    /// `_VALUE_KIND` field is added. Primary-key mode receives the requested realtime primary-key
+    /// transport schema.
     /// This schema is borrowed and remains valid only during `CreateQueryReaders`; plugins must
     /// import or copy it synchronously.
     ::ArrowSchema* read_schema;
@@ -145,8 +146,8 @@ class PAIMON_EXPORT RealtimeStore {
     ///
     /// The returned readers collectively expose every sealed row exactly once. Append-mode readers
     /// preserve write order and contain `_VALUE_KIND` followed by table write fields. Primary-key
-    /// readers use the prepared transport schema; each reader's complete stream is sorted by full
-    /// primary key then sequence number.
+    /// readers use the realtime primary-key transport schema; each reader's complete stream is
+    /// sorted by full primary key then sequence number.
     virtual Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
         const std::shared_ptr<RealtimeSegmentHandle>& segment) = 0;
 
@@ -161,11 +162,11 @@ class PAIMON_EXPORT RealtimeStore {
     ///
     /// Append-mode batches contain `_VALUE_KIND` followed by the requested fields except a
     /// duplicate `_VALUE_KIND`, and collectively expose every matching row exactly once.
-    /// Primary-key batches use the requested prepared transport schema, including nested field-ID
-    /// alignment, and may contain multiple mutations per key; each reader's complete stream is
-    /// sorted by full primary key then sequence number, and the readers collectively expose every
-    /// raw mutation exactly once. Paimon retains `view` for the lifetime of the resulting framework
-    /// reader.
+    /// Primary-key batches use the requested realtime primary-key transport schema, including
+    /// nested field-ID alignment, and may contain multiple mutations per key; each reader's
+    /// complete stream is sorted by full primary key then sequence number, and the readers
+    /// collectively expose every raw mutation exactly once. Paimon retains `view` for the lifetime
+    /// of the resulting framework reader.
     virtual Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
         const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
         const RealtimeQueryContext& context) = 0;
diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index 41fe5f6ba..7e4986ab8 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -382,7 +382,7 @@ set(PAIMON_CORE_SRCS
     core/operation/write_restore.cpp
     core/realtime/arrow_realtime_store.cpp
     core/realtime/arrow_realtime_store_factory.cpp
-    core/realtime/prepared_key_value_reader.cpp
+    core/realtime/realtime_primary_key_reader.cpp
     core/realtime/primary_key_realtime_store.cpp
     core/realtime/realtime_append_only_writer.cpp
     core/realtime/realtime_context.cpp
@@ -791,7 +791,7 @@ if(PAIMON_BUILD_TESTS)
                     core/memory/writer_memory_manager_test.cpp
                     core/realtime/arrow_realtime_store_test.cpp
                     core/realtime/primary_key_realtime_store_test.cpp
-                    core/realtime/prepared_key_value_reader_test.cpp
+                    core/realtime/realtime_primary_key_reader_test.cpp
                     core/realtime/realtime_context_test.cpp
                     core/realtime/realtime_reader_test.cpp
                     core/mergetree/levels_test.cpp
diff --git a/src/paimon/common/table/special_fields.h b/src/paimon/common/table/special_fields.h
index 0e07882a8..8908f9f0c 100644
--- a/src/paimon/common/table/special_fields.h
+++ b/src/paimon/common/table/special_fields.h
@@ -36,10 +36,6 @@ struct SpecialFields {
 
     static constexpr char KEY_FIELD_PREFIX[] = "_KEY_";
     static constexpr int32_t KEY_VALUE_SPECIAL_FIELD_COUNT = 2;
-    static constexpr int32_t kPreparedKeyValueValueKindIndex = 0;
-    static constexpr int32_t kPreparedKeyValueSequenceNumberIndex = 1;
-    static constexpr int32_t kPreparedKeyValueRealtimeOffsetIndex = 2;
-    static constexpr int32_t kPreparedKeyValueValueStartIndex = 3;
 
     static const DataField& SequenceNumber() {
         static const DataField data_field = DataField(
@@ -97,16 +93,6 @@ struct SpecialFields {
         target_fields.insert(target_fields.end(), schema->fields().begin(), schema->fields().end());
         return arrow::schema(target_fields);
     }
-
-    static std::shared_ptr<arrow::Schema> PreparedKeyValueSchema(
-        const arrow::FieldVector& value_fields) {
-        arrow::FieldVector fields = {
-            DataField::ConvertDataFieldToArrowField(ValueKind())->WithNullable(false),
-            DataField::ConvertDataFieldToArrowField(SequenceNumber())->WithNullable(false),
-            DataField::ConvertDataFieldToArrowField(RealtimeOffset())};
-        fields.insert(fields.end(), value_fields.begin(), value_fields.end());
-        return arrow::schema(std::move(fields));
-    }
 };
 
 }  // namespace paimon
diff --git a/src/paimon/common/table/special_fields_test.cpp b/src/paimon/common/table/special_fields_test.cpp
index a0a0980a5..58a025ba2 100644
--- a/src/paimon/common/table/special_fields_test.cpp
+++ b/src/paimon/common/table/special_fields_test.cpp
@@ -66,27 +66,6 @@ TEST(SpecialFieldsTest, TestKeyValueSpecialFieldCount) {
     ASSERT_EQ(SpecialFields::KEY_VALUE_SPECIAL_FIELD_COUNT, 2);
 }
 
-TEST(SpecialFieldsTest, TestPreparedKeyValueSchema) {
-    arrow::FieldVector value_fields = {arrow::field("key", arrow::int64(), false),
-                                       arrow::field("value", arrow::utf8())};
-    std::shared_ptr<arrow::Schema> schema = SpecialFields::PreparedKeyValueSchema(value_fields);
-
-    ASSERT_EQ(SpecialFields::kPreparedKeyValueValueKindIndex, 0);
-    ASSERT_EQ(SpecialFields::kPreparedKeyValueSequenceNumberIndex, 1);
-    ASSERT_EQ(SpecialFields::kPreparedKeyValueRealtimeOffsetIndex, 2);
-    ASSERT_EQ(SpecialFields::kPreparedKeyValueValueStartIndex, 3);
-    ASSERT_EQ(schema->field(0)->name(), "_VALUE_KIND");
-    ASSERT_EQ(schema->field(1)->name(), "_SEQUENCE_NUMBER");
-    ASSERT_EQ(schema->field(2)->name(), "_REALTIME_OFFSET");
-    ASSERT_EQ(schema->field(3)->name(), "key");
-    ASSERT_EQ(schema->field(4)->name(), "value");
-    ASSERT_FALSE(schema->field(0)->nullable());
-    ASSERT_FALSE(schema->field(1)->nullable());
-    ASSERT_EQ(schema->field(2)->nullable(), SpecialFields::RealtimeOffset().Nullable());
-    ASSERT_FALSE(schema->field(3)->nullable());
-    ASSERT_TRUE(schema->field(4)->nullable());
-}
-
 TEST(SpecialFieldsTest, TestIsSystemField) {
     ASSERT_TRUE(SpecialFields::IsSystemField("_SEQUENCE_NUMBER"));
     ASSERT_TRUE(SpecialFields::IsSystemField("_VALUE_KIND"));
diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index a9395c9ab..d484b28ee 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -35,7 +35,7 @@
 #include "paimon/common/utils/fields_comparator.h"
 #include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
 #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
-#include "paimon/core/realtime/prepared_key_value_reader.h"
+#include "paimon/core/realtime/realtime_primary_key_reader.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/realtime/offset_range.h"
 #include "paimon/testing/mock/mock_file_batch_reader.h"
diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp
index 86f432998..152a4ed01 100644
--- a/src/paimon/core/operation/key_value_file_store_write.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write.cpp
@@ -35,6 +35,7 @@
 #include "paimon/core/operation/file_store_scan.h"
 #include "paimon/core/operation/key_value_file_store_scan.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
+#include "paimon/core/realtime/realtime_primary_key_reader.h"
 #include "paimon/core/realtime/realtime_primary_key_writer.h"
 #include "paimon/core/schema/table_schema.h"
 #include "paimon/core/utils/file_store_path_factory.h"
@@ -124,7 +125,7 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
     std::shared_ptr<CompactManager> compact_manager;
     std::shared_ptr<RealtimeContextImpl> realtime_context_impl;
     std::optional<RealtimeStoreState> realtime_store_state;
-    std::shared_ptr<arrow::Schema> prepared_schema;
+    std::shared_ptr<arrow::Schema> transport_schema;
     if (realtime_context_) {
         std::vector<std::pair<std::string, std::string>> partition_values;
         PAIMON_ASSIGN_OR_RAISE(partition_values,
@@ -132,10 +133,10 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
         partition_map =
             std::map<std::string, std::string>(partition_values.begin(), partition_values.end());
         PAIMON_ASSIGN_OR_RAISE(realtime_context_impl, RealtimeContextImpl::Cast(realtime_context_));
-        prepared_schema = SpecialFields::PreparedKeyValueSchema(schema_->fields());
+        transport_schema = RealtimePrimaryKeyLayout::CreateSchema(schema_->fields());
         auto c_write_schema = std::make_unique<ArrowSchema>();
         PAIMON_RETURN_NOT_OK_FROM_ARROW(
-            arrow::ExportSchema(*prepared_schema, c_write_schema.get()));
+            arrow::ExportSchema(*transport_schema, c_write_schema.get()));
         PAIMON_ASSIGN_OR_RAISE(
             RealtimeStoreState store_state,
             realtime_context_impl->GetOrCreateRealtimeStore(
@@ -164,9 +165,10 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
     if (!realtime_context_) {
         return std::shared_ptr<BatchWriter>(std::move(writer));
     }
-    return RealtimePrimaryKeyWriter::Create(
-        partition_map, bucket, schema_, prepared_schema, trimmed_primary_keys, key_comparator_,
-        realtime_context_impl, realtime_store_state.value(), restore_max_seq_number, writer, pool_);
+    return RealtimePrimaryKeyWriter::Create(partition_map, bucket, schema_, transport_schema,
+                                            trimmed_primary_keys, key_comparator_, options_,
+                                            realtime_context_impl, realtime_store_state.value(),
+                                            restore_max_seq_number, writer, pool_);
 }
 
 Status KeyValueFileStoreWrite::RefreshCommittedSnapshot(int64_t snapshot_id) {
diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp
index fc6bfff11..01ba8459d 100644
--- a/src/paimon/core/operation/key_value_file_store_write_test.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp
@@ -248,7 +248,8 @@ class KeyValueFileStoreWriteTest : public ::testing::Test {
     }
 
     Result<std::vector<std::tuple<int8_t, int64_t, std::string, int64_t, int64_t>>>
-    ReadPreparedRows(const std::shared_ptr<RealtimeContext>& realtime_context) const {
+    ReadRealtimePrimaryKeyTransportRows(
+        const std::shared_ptr<RealtimeContext>& realtime_context) const {
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> context,
                                RealtimeContextImpl::Cast(realtime_context));
         PAIMON_ASSIGN_OR_RAISE(std::vector<RealtimePartitionBucketView> views,
@@ -256,7 +257,7 @@ class KeyValueFileStoreWriteTest : public ::testing::Test {
         if (views.size() != 1) {
             return Status::Invalid("expected exactly one real-time store");
         }
-        std::shared_ptr<arrow::Schema> prepared_schema = arrow::schema({
+        std::shared_ptr<arrow::Schema> transport_schema = arrow::schema({
             DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())
                 ->WithNullable(false),
             DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
@@ -268,7 +269,7 @@ class KeyValueFileStoreWriteTest : public ::testing::Test {
                 DataField(1, arrow::field("value", arrow::utf8()))),
         });
         auto c_schema = std::make_unique<ArrowSchema>();
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*prepared_schema, c_schema.get()));
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*transport_schema, c_schema.get()));
         RealtimeQueryContext query_context{c_schema.get(), nullptr, false};
         PAIMON_ASSIGN_OR_RAISE(
             std::vector<std::unique_ptr<BatchReader>> readers,
@@ -286,7 +287,7 @@ class KeyValueFileStoreWriteTest : public ::testing::Test {
                 std::shared_ptr<arrow::StructArray> values =
                     std::dynamic_pointer_cast<arrow::StructArray>(array);
                 if (!values || values->num_fields() != 5) {
-                    return Status::Invalid("unexpected prepared real-time batch");
+                    return Status::Invalid("unexpected realtime primary-key transport batch");
                 }
                 std::shared_ptr<arrow::Int8Array> row_kinds =
                     std::dynamic_pointer_cast<arrow::Int8Array>(values->field(0));
@@ -299,7 +300,7 @@ class KeyValueFileStoreWriteTest : public ::testing::Test {
                 std::shared_ptr<arrow::StringArray> payloads =
                     std::dynamic_pointer_cast<arrow::StringArray>(values->field(4));
                 if (!row_kinds || !sequences || !offsets || !ids || !payloads) {
-                    return Status::Invalid("unexpected prepared real-time column type");
+                    return Status::Invalid("unexpected realtime primary-key transport column type");
                 }
                 for (int64_t row = 0; row < values->length(); ++row) {
                     rows.emplace_back(row_kinds->Value(row), ids->Value(row),
@@ -456,12 +457,13 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeWrite) {
                   {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::DELETE,
                    RecordBatch::RowKind::UPDATE_AFTER});
     ASSERT_OK(writer->Write(std::move(batch)));
-    using PreparedRow = std::tuple<int8_t, int64_t, std::string, int64_t, int64_t>;
-    ASSERT_OK_AND_ASSIGN(std::vector<PreparedRow> prepared_rows,
-                         ReadPreparedRows(realtime_context));
-    ASSERT_EQ(
-        (std::vector<PreparedRow>{{0, 1, "old", 0, 0}, {2, 1, "new", 2, 2}, {3, 2, "two", 1, 1}}),
-        prepared_rows);
+    using RealtimePrimaryKeyTransportRow =
+        std::tuple<int8_t, int64_t, std::string, int64_t, int64_t>;
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimePrimaryKeyTransportRow> transport_rows,
+                         ReadRealtimePrimaryKeyTransportRows(realtime_context));
+    ASSERT_EQ((std::vector<RealtimePrimaryKeyTransportRow>{
+                  {0, 1, "old", 0, 0}, {2, 1, "new", 2, 2}, {3, 2, "two", 1, 1}}),
+              transport_rows);
     ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progresses,
                          writer->PrepareCommitWithProgress(0));
     ASSERT_EQ(1, progresses.size());
@@ -508,10 +510,11 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) {
     ASSERT_GT(pool->allocation_count, allocations_before_write);
     ASSERT_OK(writer->Close());
     writer.reset();
-    using PreparedRow = std::tuple<int8_t, int64_t, std::string, int64_t, int64_t>;
-    ASSERT_OK_AND_ASSIGN(std::vector<PreparedRow> retained_rows,
-                         ReadPreparedRows(realtime_context));
-    ASSERT_EQ((std::vector<PreparedRow>{{0, 1, "one", 0, 0}}), retained_rows);
+    using RealtimePrimaryKeyTransportRow =
+        std::tuple<int8_t, int64_t, std::string, int64_t, int64_t>;
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimePrimaryKeyTransportRow> retained_rows,
+                         ReadRealtimePrimaryKeyTransportRows(realtime_context));
+    ASSERT_EQ((std::vector<RealtimePrimaryKeyTransportRow>{{0, 1, "one", 0, 0}}), retained_rows);
 
     std::shared_ptr<TestingMemoryPool> rejecting_pool = std::make_shared<TestingMemoryPool>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> rejecting_context,
@@ -531,8 +534,8 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) {
     ASSERT_NOK_WITH_MSG(rejecting_writer->Write(MakeBatch(schema, R"([[2, "two"]])")),
                         "Out of memory");
     ASSERT_GT(rejecting_pool->allocation_count, rejecting_allocations_before_write);
-    ASSERT_OK_AND_ASSIGN(std::vector<PreparedRow> rejected_rows,
-                         ReadPreparedRows(rejecting_context));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimePrimaryKeyTransportRow> rejected_rows,
+                         ReadRealtimePrimaryKeyTransportRows(rejecting_context));
     ASSERT_TRUE(rejected_rows.empty());
     ASSERT_OK(rejecting_writer->Close());
 }
@@ -587,15 +590,18 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeLimits) {
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                          FileStoreWrite::Create(std::move(write_context)));
     ASSERT_OK(writer->Write(MakeBatch(schema, R"([[1, "legal"]])")));
-    using PreparedRow = std::tuple<int8_t, int64_t, std::string, int64_t, int64_t>;
-    ASSERT_OK_AND_ASSIGN(std::vector<PreparedRow> prepared_rows,
-                         ReadPreparedRows(realtime_context));
-    ASSERT_EQ((std::vector<PreparedRow>{{0, 1, "legal", max - 1, max - 1}}), prepared_rows);
+    using RealtimePrimaryKeyTransportRow =
+        std::tuple<int8_t, int64_t, std::string, int64_t, int64_t>;
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimePrimaryKeyTransportRow> transport_rows,
+                         ReadRealtimePrimaryKeyTransportRows(realtime_context));
+    ASSERT_EQ((std::vector<RealtimePrimaryKeyTransportRow>{{0, 1, "legal", max - 1, max - 1}}),
+              transport_rows);
 
     ASSERT_NOK_WITH_MSG(writer->Write(MakeBatch(schema, R"([[2, "overflow"]])")),
                         "real-time offset range exceeds INT64_MAX");
-    ASSERT_OK_AND_ASSIGN(prepared_rows, ReadPreparedRows(realtime_context));
-    ASSERT_EQ((std::vector<PreparedRow>{{0, 1, "legal", max - 1, max - 1}}), prepared_rows);
+    ASSERT_OK_AND_ASSIGN(transport_rows, ReadRealtimePrimaryKeyTransportRows(realtime_context));
+    ASSERT_EQ((std::vector<RealtimePrimaryKeyTransportRow>{{0, 1, "legal", max - 1, max - 1}}),
+              transport_rows);
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context_impl,
                          RealtimeContextImpl::Cast(realtime_context));
     ASSERT_OK_AND_ASSIGN(std::vector<RealtimePartitionBucketView> views,
diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp
index 3b64ed20e..3bcc705aa 100644
--- a/src/paimon/core/operation/merge_file_split_read.cpp
+++ b/src/paimon/core/operation/merge_file_split_read.cpp
@@ -677,6 +677,7 @@ Result<std::unique_ptr<BatchReader>> MergeFileSplitRead::CreateProjectedReader(
     if (!force_keep_delete_) {
         sort_merge_reader = std::make_unique<DropDeleteReader>(std::move(sort_merge_reader));
     }
+    // KeyValueProjectionReader converts KeyValue objects to arrow array according to projection
     std::unique_ptr<BatchReader> projection_reader;
     if (!context_->EnableMultiThreadRowToBatch()) {
         PAIMON_ASSIGN_OR_RAISE(
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index a22ba8fed..421eb0c8d 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -33,7 +33,6 @@
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/scope_guard.h"
-#include "paimon/core/realtime/prepared_key_value_reader.h"
 #include "paimon/core/utils/nested_projection_utils.h"
 #include "paimon/macros.h"
 #include "paimon/memory/memory_pool.h"
@@ -138,9 +137,9 @@ class StoredBatchReader final : public BatchReader {
 
 class PrimaryKeyRealtimeStore::Impl {
  public:
-    Impl(std::shared_ptr<arrow::Schema> prepared_schema,
+    Impl(std::shared_ptr<arrow::Schema> transport_schema,
          std::shared_ptr<arrow::MemoryPool> arrow_pool)
-        : prepared_schema_(std::move(prepared_schema)), arrow_pool_(std::move(arrow_pool)) {}
+        : transport_schema_(std::move(transport_schema)), arrow_pool_(std::move(arrow_pool)) {}
 
     Status Write(RealtimeWriteBatch&& write_batch) {
         if (!write_batch.batch || !write_batch.batch->GetData()) {
@@ -154,15 +153,15 @@ class PrimaryKeyRealtimeStore::Impl {
         PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
             std::shared_ptr<arrow::Array> array,
             arrow::ImportArray(write_batch.batch->GetData(),
-                               arrow::struct_(prepared_schema_->fields())));
+                               arrow::struct_(transport_schema_->fields())));
         if (!array || array->type_id() != arrow::Type::STRUCT) {
-            return Status::Invalid("PK real-time prepared batch is not a StructArray");
+            return Status::Invalid("PK real-time transport batch is not a StructArray");
         }
-        std::shared_ptr<arrow::StructArray> prepared =
+        std::shared_ptr<arrow::StructArray> transport =
             checked_pointer_cast<arrow::StructArray>(array);
         std::lock_guard<std::mutex> lock(mutex_);
-        building_.push_back(StoredBatch{prepared, write_batch.offset_range,
-                                        ArrowUtils::GetArrayMemoryUsage(prepared->data())});
+        building_.push_back(StoredBatch{transport, write_batch.offset_range,
+                                        ArrowUtils::GetArrayMemoryUsage(transport->data())});
         building_memory_usage_ += building_.back().memory_usage;
         return Status::OK();
     }
@@ -218,7 +217,6 @@ class PrimaryKeyRealtimeStore::Impl {
         }
         PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> read_schema,
                                           arrow::ImportSchema(context.read_schema));
-        PAIMON_RETURN_NOT_OK(PreparedKeyValueReaderFactory::ValidateTransportSchema(read_schema));
         std::vector<std::unique_ptr<BatchReader>> readers;
         for (const std::shared_ptr<Segment>& segment : typed->Segments()) {
             for (const StoredBatch& batch : segment->Batches()) {
@@ -260,7 +258,7 @@ class PrimaryKeyRealtimeStore::Impl {
     }
 
  private:
-    std::shared_ptr<arrow::Schema> prepared_schema_;
+    std::shared_ptr<arrow::Schema> transport_schema_;
     std::shared_ptr<arrow::MemoryPool> arrow_pool_;
     mutable std::mutex mutex_;
     std::vector<StoredBatch> building_;
@@ -273,15 +271,14 @@ PrimaryKeyRealtimeStore::PrimaryKeyRealtimeStore(std::unique_ptr<Impl>&& impl)
 PrimaryKeyRealtimeStore::~PrimaryKeyRealtimeStore() = default;
 
 Result<std::shared_ptr<PrimaryKeyRealtimeStore>> PrimaryKeyRealtimeStore::Create(
-    const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::shared_ptr<arrow::Schema>& transport_schema,
     const std::shared_ptr<MemoryPool>& memory_pool) {
-    PAIMON_RETURN_NOT_OK(PreparedKeyValueReaderFactory::ValidateTransportSchema(prepared_schema));
     if (!memory_pool) {
         return Status::Invalid("PK real-time store memory pool is null");
     }
     std::shared_ptr<arrow::MemoryPool> arrow_pool = GetArrowPool(memory_pool);
     return std::shared_ptr<PrimaryKeyRealtimeStore>(new PrimaryKeyRealtimeStore(
-        std::make_unique<Impl>(prepared_schema, std::move(arrow_pool))));
+        std::make_unique<Impl>(transport_schema, std::move(arrow_pool))));
 }
 Status PrimaryKeyRealtimeStore::Write(RealtimeWriteBatch&& batch) {
     return impl_->Write(std::move(batch));
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.h b/src/paimon/core/realtime/primary_key_realtime_store.h
index 01e1926ab..46c0fe8f7 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.h
+++ b/src/paimon/core/realtime/primary_key_realtime_store.h
@@ -35,7 +35,7 @@ class MemoryPool;
 class PrimaryKeyRealtimeStore final : public RealtimeStore {
  public:
     static Result<std::shared_ptr<PrimaryKeyRealtimeStore>> Create(
-        const std::shared_ptr<arrow::Schema>& prepared_schema,
+        const std::shared_ptr<arrow::Schema>& transport_schema,
         const std::shared_ptr<MemoryPool>& memory_pool);
 
     ~PrimaryKeyRealtimeStore() override;
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index 46d9a8e7d..301cc92a9 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -49,7 +49,7 @@ std::shared_ptr<arrow::Field> FieldWithId(const std::string& name,
         ->WithNullable(nullable);
 }
 
-std::shared_ptr<arrow::Schema> PreparedSchema() {
+std::shared_ptr<arrow::Schema> TransportSchema() {
     return arrow::schema(
         {DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
          DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
@@ -60,7 +60,7 @@ std::shared_ptr<arrow::Schema> PreparedSchema() {
              DataField(1, arrow::field("value", arrow::utf8())))});
 }
 
-std::shared_ptr<arrow::Schema> NestedPreparedSchema() {
+std::shared_ptr<arrow::Schema> NestedTransportSchema() {
     return arrow::schema(
         {DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
          DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
@@ -76,7 +76,7 @@ std::shared_ptr<arrow::Schema> NestedPreparedSchema() {
 
 std::unique_ptr<RecordBatch> MakeBatch(const std::string& json) {
     std::shared_ptr<arrow::Array> array =
-        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(PreparedSchema()->fields()), json)
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(TransportSchema()->fields()), json)
             .ValueOrDie();
     auto c_array = std::make_unique<ArrowArray>();
     EXPECT_TRUE(arrow::ExportArray(*array, c_array.get()).ok());
@@ -157,7 +157,7 @@ class TestingMemoryPool final : public MemoryPool {
 
 TEST(PrimaryKeyRealtimeStoreTest, TestWriteAndSealValidation) {
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(PreparedSchema(), GetDefaultPool()));
+                         PrimaryKeyRealtimeStore::Create(TransportSchema(), GetDefaultPool()));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
                          store->SealForCommit());
     ASSERT_FALSE(segment.has_value());
@@ -180,37 +180,9 @@ TEST(PrimaryKeyRealtimeStoreTest, TestWriteAndSealValidation) {
         RealtimeWriteBatch{MakeBatch(R"([[0, 4, 3, 4, "four"]])"), OffsetRange(3, 4)}));
 }
 
-TEST(PrimaryKeyRealtimeStoreTest, TestBadTransportPrefix) {
-    const std::shared_ptr<arrow::Schema> valid = PreparedSchema();
-    std::vector<arrow::FieldVector> invalid_fields;
-
-    arrow::FieldVector wrong_type = valid->fields();
-    wrong_type[0] = DataField::ConvertDataFieldToArrowField(
-                        DataField(SpecialFields::ValueKind().Id(),
-                                  arrow::field("_VALUE_KIND", arrow::int32(), false)))
-                        ->WithNullable(false);
-    invalid_fields.push_back(std::move(wrong_type));
-
-    arrow::FieldVector nullable_sequence = valid->fields();
-    nullable_sequence[1] = nullable_sequence[1]->WithNullable(true);
-    invalid_fields.push_back(std::move(nullable_sequence));
-
-    arrow::FieldVector wrong_offset_id = valid->fields();
-    wrong_offset_id[2] = DataField::ConvertDataFieldToArrowField(
-                             DataField(99, arrow::field("_REALTIME_OFFSET", arrow::int64(), false)))
-                             ->WithNullable(false);
-    invalid_fields.push_back(std::move(wrong_offset_id));
-
-    for (const arrow::FieldVector& fields : invalid_fields) {
-        ASSERT_NOK_WITH_MSG(
-            PrimaryKeyRealtimeStore::Create(arrow::schema(fields), GetDefaultPool()),
-            "prepared schema field");
-    }
-}
-
 TEST(PrimaryKeyRealtimeStoreTest, TestCommitReaderPerStoredBatch) {
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(PreparedSchema(), GetDefaultPool()));
+                         PrimaryKeyRealtimeStore::Create(TransportSchema(), GetDefaultPool()));
     ASSERT_OK(store->Write(RealtimeWriteBatch{
         MakeBatch(R"([[1, 6, 1, 1, "before"], [0, 5, 0, 3, "three"]])"), OffsetRange(0, 2)}));
     ASSERT_OK(store->Write(
@@ -264,7 +236,7 @@ void AssertSlicedBatch(BatchReader* reader) {
 }
 
 TEST(PrimaryKeyRealtimeStoreTest, TestSlicedReadersExportZeroOffsets) {
-    std::shared_ptr<arrow::Schema> schema = NestedPreparedSchema();
+    std::shared_ptr<arrow::Schema> schema = NestedTransportSchema();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
                          PrimaryKeyRealtimeStore::Create(schema, GetDefaultPool()));
     ASSERT_OK(store->Write(RealtimeWriteBatch{
@@ -293,7 +265,7 @@ TEST(PrimaryKeyRealtimeStoreTest, TestSlicedReadersExportZeroOffsets) {
 
 TEST(PrimaryKeyRealtimeStoreTest, TestCloseUnreadBatchReaders) {
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(PreparedSchema(), GetDefaultPool()));
+                         PrimaryKeyRealtimeStore::Create(TransportSchema(), GetDefaultPool()));
     ASSERT_OK(
         store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 10, 0, 1, "a"]])"), OffsetRange(0, 1)}));
     ASSERT_OK(
@@ -313,7 +285,7 @@ TEST(PrimaryKeyRealtimeStoreTest, TestCloseUnreadBatchReaders) {
 
 TEST(PrimaryKeyRealtimeStoreTest, TestReclaimKeepsReadView) {
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(PreparedSchema(), GetDefaultPool()));
+                         PrimaryKeyRealtimeStore::Create(TransportSchema(), GetDefaultPool()));
     ASSERT_OK(
         store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 0, 4, 1, "one"]])"), OffsetRange(4, 5)}));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
@@ -349,7 +321,7 @@ TEST(PrimaryKeyRealtimeStoreTest, TestReclaimKeepsReadView) {
     ASSERT_FALSE(current_view->GetOffsetRange().has_value());
 
     auto c_schema = std::make_unique<ArrowSchema>();
-    ASSERT_TRUE(arrow::ExportSchema(*PreparedSchema(), c_schema.get()).ok());
+    ASSERT_TRUE(arrow::ExportSchema(*TransportSchema(), c_schema.get()).ok());
     RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
                                  /*enable_predicate_pushdown=*/false};
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
@@ -362,7 +334,7 @@ TEST(PrimaryKeyRealtimeStoreTest, TestReclaimKeepsReadView) {
 
 TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderPerStoredBatch) {
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(PreparedSchema(), GetDefaultPool()));
+                         PrimaryKeyRealtimeStore::Create(TransportSchema(), GetDefaultPool()));
     ASSERT_OK(
         store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 1, 0, 2, "two"]])"), OffsetRange(0, 1)}));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
@@ -372,7 +344,7 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderPerStoredBatch) {
         store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 2, 1, 1, "one"]])"), OffsetRange(1, 2)}));
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
     auto c_schema = std::make_unique<ArrowSchema>();
-    ASSERT_TRUE(arrow::ExportSchema(*PreparedSchema(), c_schema.get()).ok());
+    ASSERT_TRUE(arrow::ExportSchema(*TransportSchema(), c_schema.get()).ok());
     RealtimeQueryContext context{/*read_schema=*/c_schema.get(), /*predicate=*/nullptr,
                                  /*enable_predicate_pushdown=*/false};
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
@@ -384,7 +356,7 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderPerStoredBatch) {
 }
 
 TEST(PrimaryKeyRealtimeStoreTest, TestQueryPoolOutlivesStoreReaderAndExport) {
-    const std::shared_ptr<arrow::Schema> stored_schema = PreparedSchema();
+    const std::shared_ptr<arrow::Schema> stored_schema = TransportSchema();
     std::shared_ptr<TestingMemoryPool> pool = std::make_shared<TestingMemoryPool>();
     std::weak_ptr<TestingMemoryPool> pool_lifetime = pool;
     auto write_schema = std::make_unique<ArrowSchema>();
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/realtime_primary_key_reader.cpp
similarity index 65%
rename from src/paimon/core/realtime/prepared_key_value_reader.cpp
rename to src/paimon/core/realtime/realtime_primary_key_reader.cpp
index d5e29da52..19ba0a985 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_reader.cpp
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-#include "paimon/core/realtime/prepared_key_value_reader.h"
+#include "paimon/core/realtime/realtime_primary_key_reader.h"
 
 #include <cstdint>
 #include <memory>
@@ -122,19 +122,20 @@ class RealtimeOffsetCoverage {
     size_t finished_reader_count_ = 0;
 };
 
-Status CheckPreparedField(const std::shared_ptr<arrow::Schema>& schema, int32_t field_idx,
-                          const DataField& expected_field) {
+Status CheckTransportField(const std::shared_ptr<arrow::Schema>& schema, int32_t field_idx,
+                           const DataField& expected_field) {
     if (schema->num_fields() <= field_idx) {
-        return Status::Invalid(fmt::format("prepared schema missing transport field {} at index {}",
-                                           expected_field.Name(), field_idx));
+        return Status::Invalid(
+            fmt::format("realtime primary-key transport schema is missing field {} at index {}",
+                        expected_field.Name(), field_idx));
     }
     const std::shared_ptr<arrow::Field>& field = schema->field(field_idx);
     PAIMON_ASSIGN_OR_RAISE(int32_t field_id, NestedProjectionUtils::GetPaimonFieldId(field));
     if (field->name() != expected_field.Name() || !field->type()->Equals(*expected_field.Type()) ||
         field->nullable() || field_id != expected_field.Id()) {
         return Status::Invalid(fmt::format(
-            "prepared schema field {} must be non-null {}:{} with field id {}, got {}:{} "
-            "nullable={} field id {}",
+            "realtime primary-key transport schema field {} must be non-null {}:{} with field id "
+            "{}, got {}:{} nullable={} field id {}",
             field_idx, expected_field.Name(), expected_field.Type()->ToString(),
             expected_field.Id(), field->name(), field->type()->ToString(), field->nullable(),
             field_id));
@@ -143,7 +144,7 @@ Status CheckPreparedField(const std::shared_ptr<arrow::Schema>& schema, int32_t
 }
 
 Result<std::vector<int32_t>> ResolveFieldIndexes(
-    const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::shared_ptr<arrow::Schema>& transport_schema,
     const std::unordered_map<int32_t, int32_t>& field_indexes,
     const std::shared_ptr<arrow::Schema>& row_schema) {
     std::vector<int32_t> result;
@@ -153,50 +154,49 @@ Result<std::vector<int32_t>> ResolveFieldIndexes(
                                NestedProjectionUtils::GetPaimonFieldId(row_field));
         auto field_index = field_indexes.find(field_id);
         if (field_index == field_indexes.end()) {
-            return Status::Invalid(
-                fmt::format("cannot find field id {} in prepared schema", field_id));
+            return Status::Invalid(fmt::format(
+                "cannot find field id {} in realtime primary-key transport schema", field_id));
         }
-        const std::shared_ptr<arrow::Field>& prepared_field =
-            prepared_schema->field(field_index->second);
-        if (!prepared_field->type()->Equals(row_field->type())) {
+        const std::shared_ptr<arrow::Field>& transport_field =
+            transport_schema->field(field_index->second);
+        if (!transport_field->type()->Equals(row_field->type())) {
             return Status::Invalid(fmt::format(
-                "prepared field id {} type {} does not match row "
-                "type {}",
-                field_id, prepared_field->type()->ToString(), row_field->type()->ToString()));
+                "realtime primary-key transport field id {} type {} does not match row type {}",
+                field_id, transport_field->type()->ToString(), row_field->type()->ToString()));
         }
         result.push_back(field_index->second);
     }
     return result;
 }
 
-class PreparedReaderPlan {
+class RealtimePrimaryKeyReaderPlan {
  public:
-    static Result<std::shared_ptr<const PreparedReaderPlan>> Create(
-        const std::shared_ptr<arrow::Schema>& prepared_schema,
+    static Result<std::shared_ptr<const RealtimePrimaryKeyReaderPlan>> Create(
+        const std::shared_ptr<arrow::Schema>& transport_schema,
         const std::shared_ptr<arrow::Schema>& key_schema,
         const std::shared_ptr<arrow::Schema>& value_schema) {
         std::unordered_map<int32_t, int32_t> field_indexes;
-        field_indexes.reserve(prepared_schema->num_fields() -
-                              SpecialFields::kPreparedKeyValueValueStartIndex);
-        for (int32_t i = SpecialFields::kPreparedKeyValueValueStartIndex;
-             i < prepared_schema->num_fields(); ++i) {
+        field_indexes.reserve(transport_schema->num_fields() -
+                              RealtimePrimaryKeyLayout::kValueStartIndex);
+        for (int32_t i = RealtimePrimaryKeyLayout::kValueStartIndex;
+             i < transport_schema->num_fields(); ++i) {
             PAIMON_ASSIGN_OR_RAISE(int32_t field_id, NestedProjectionUtils::GetPaimonFieldId(
-                                                         prepared_schema->field(i)));
+                                                         transport_schema->field(i)));
             if (!field_indexes.emplace(field_id, i).second) {
-                return Status::Invalid(
-                    fmt::format("duplicate field id {} in prepared schema", field_id));
+                return Status::Invalid(fmt::format(
+                    "duplicate field id {} in realtime primary-key transport schema", field_id));
             }
         }
         PAIMON_ASSIGN_OR_RAISE(std::vector<int32_t> key_field_indexes,
-                               ResolveFieldIndexes(prepared_schema, field_indexes, key_schema));
+                               ResolveFieldIndexes(transport_schema, field_indexes, key_schema));
         PAIMON_ASSIGN_OR_RAISE(std::vector<int32_t> value_field_indexes,
-                               ResolveFieldIndexes(prepared_schema, field_indexes, value_schema));
-        return std::shared_ptr<const PreparedReaderPlan>(new PreparedReaderPlan(
-            prepared_schema, std::move(key_field_indexes), std::move(value_field_indexes)));
+                               ResolveFieldIndexes(transport_schema, field_indexes, value_schema));
+        return std::shared_ptr<const RealtimePrimaryKeyReaderPlan>(new RealtimePrimaryKeyReaderPlan(
+            transport_schema, std::move(key_field_indexes), std::move(value_field_indexes)));
     }
 
-    const std::shared_ptr<arrow::Schema>& PreparedSchema() const {
-        return prepared_schema_;
+    const std::shared_ptr<arrow::Schema>& TransportSchema() const {
+        return transport_schema_;
     }
 
     const std::vector<int32_t>& KeyFieldIndexes() const {
@@ -208,24 +208,25 @@ class PreparedReaderPlan {
     }
 
  private:
-    PreparedReaderPlan(const std::shared_ptr<arrow::Schema>& schema,
-                       std::vector<int32_t>&& key_indexes, std::vector<int32_t>&& value_indexes)
-        : prepared_schema_(schema),
+    RealtimePrimaryKeyReaderPlan(const std::shared_ptr<arrow::Schema>& schema,
+                                 std::vector<int32_t>&& key_indexes,
+                                 std::vector<int32_t>&& value_indexes)
+        : transport_schema_(schema),
           key_field_indexes_(std::move(key_indexes)),
           value_field_indexes_(std::move(value_indexes)) {}
 
-    const std::shared_ptr<arrow::Schema> prepared_schema_;
+    const std::shared_ptr<arrow::Schema> transport_schema_;
     const std::vector<int32_t> key_field_indexes_;
     const std::vector<int32_t> value_field_indexes_;
 };
 
-class PreparedKeyValueReader final : public KeyValueRecordReader {
+class RealtimePrimaryKeyReader final : public KeyValueRecordReader {
  public:
-    PreparedKeyValueReader(std::unique_ptr<BatchReader>&& reader,
-                           const std::shared_ptr<const PreparedReaderPlan>& plan,
-                           const std::optional<OffsetRange>& visible_offsets,
-                           const std::shared_ptr<MemoryPool>& pool,
-                           const std::shared_ptr<RealtimeOffsetCoverage>& offset_coverage)
+    RealtimePrimaryKeyReader(std::unique_ptr<BatchReader>&& reader,
+                             const std::shared_ptr<const RealtimePrimaryKeyReaderPlan>& plan,
+                             const std::optional<OffsetRange>& visible_offsets,
+                             const std::shared_ptr<MemoryPool>& pool,
+                             const std::shared_ptr<RealtimeOffsetCoverage>& offset_coverage)
         : reader_(std::move(reader)),
           plan_(plan),
           visible_offsets_(visible_offsets),
@@ -234,7 +235,7 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
 
     class Iterator final : public KeyValueRecordReader::Iterator {
      public:
-        explicit Iterator(PreparedKeyValueReader* reader) : reader_(reader) {}
+        explicit Iterator(RealtimePrimaryKeyReader* reader) : reader_(reader) {}
 
         Result<bool> HasNext() const override {
             return cursor_ < reader_->RowCount();
@@ -242,7 +243,7 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
 
         Result<KeyValue> Next() override {
             if (cursor_ >= reader_->RowCount()) {
-                return Status::Invalid("No more prepared key values in current iterator");
+                return Status::Invalid("No more realtime primary-key values in current iterator");
             }
             const int64_t row = reader_->RowAt(cursor_);
             std::shared_ptr<InternalRow> key =
@@ -257,7 +258,7 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
         }
 
      private:
-        PreparedKeyValueReader* reader_;
+        RealtimePrimaryKeyReader* reader_;
         int64_t cursor_ = 0;
     };
 
@@ -278,13 +279,8 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
     Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatchImpl() {
         while (true) {
             ResetBatchState();
-            BatchReader::ReadBatchWithBitmap batch_with_bitmap;
-            if (visible_offsets_.has_value()) {
-                PAIMON_ASSIGN_OR_RAISE(batch_with_bitmap, reader_->NextBatchWithBitmap());
-            } else {
-                PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader_->NextBatch());
-                batch_with_bitmap.first = std::move(batch);
-            }
+            PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatchWithBitmap batch_with_bitmap,
+                                   reader_->NextBatchWithBitmap());
             if (BatchReader::IsEofBatch(batch_with_bitmap)) {
                 if (offset_coverage_ && !offset_coverage_finished_) {
                     offset_coverage_finished_ = true;
@@ -297,23 +293,24 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
             PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> arrow_array,
                                               arrow::ImportArray(c_array.get(), c_schema.get()));
             if (!arrow_array || arrow_array->type_id() != arrow::Type::STRUCT) {
-                return Status::Invalid("cannot cast prepared batch to StructArray");
+                return Status::Invalid(
+                    "cannot cast realtime primary-key transport batch to StructArray");
             }
             std::shared_ptr<arrow::StructArray> data_batch =
                 checked_pointer_cast<arrow::StructArray>(arrow_array);
-            PAIMON_RETURN_NOT_OK(ValidatePreparedBatch(data_batch));
+            PAIMON_RETURN_NOT_OK(ValidateTransportBatch(data_batch));
 
             std::shared_ptr<arrow::NumericArray<arrow::Int64Type>> offset_array =
                 checked_pointer_cast<arrow::NumericArray<arrow::Int64Type>>(
-                    data_batch->field(SpecialFields::kPreparedKeyValueRealtimeOffsetIndex));
+                    data_batch->field(RealtimePrimaryKeyLayout::kRealtimeOffsetIndex));
             if (offset_coverage_) {
                 PAIMON_RETURN_NOT_OK(offset_coverage_->Add(*offset_array));
             }
 
             row_kind_array_ = checked_pointer_cast<arrow::NumericArray<arrow::Int8Type>>(
-                data_batch->field(SpecialFields::kPreparedKeyValueValueKindIndex));
+                data_batch->field(RealtimePrimaryKeyLayout::kValueKindIndex));
             sequence_number_array_ = checked_pointer_cast<arrow::NumericArray<arrow::Int64Type>>(
-                data_batch->field(SpecialFields::kPreparedKeyValueSequenceNumberIndex));
+                data_batch->field(RealtimePrimaryKeyLayout::kSequenceNumberIndex));
             arrow::ArrayVector key_fields;
             key_fields.reserve(plan_->KeyFieldIndexes().size());
             for (int32_t index : plan_->KeyFieldIndexes()) {
@@ -336,53 +333,48 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
         }
     }
 
-    Status ValidatePreparedBatch(const std::shared_ptr<arrow::StructArray>& data_batch) const {
-        if (data_batch->num_fields() != plan_->PreparedSchema()->num_fields()) {
+    Status ValidateTransportBatch(const std::shared_ptr<arrow::StructArray>& data_batch) const {
+        if (data_batch->num_fields() != plan_->TransportSchema()->num_fields()) {
             return Status::Invalid(fmt::format(
-                "prepared batch field count {} does not match prepared schema field count {}",
-                data_batch->num_fields(), plan_->PreparedSchema()->num_fields()));
+                "realtime primary-key transport batch field count {} does not match schema field "
+                "count {}",
+                data_batch->num_fields(), plan_->TransportSchema()->num_fields()));
         }
         const arrow::FieldVector& batch_fields = data_batch->type()->fields();
         for (int32_t i = 0; i < data_batch->num_fields(); ++i) {
-            if (!batch_fields[i]->Equals(plan_->PreparedSchema()->field(i), true)) {
+            if (!batch_fields[i]->Equals(plan_->TransportSchema()->field(i), true)) {
                 return Status::Invalid(fmt::format(
-                    "prepared batch field {} does not match declared prepared schema", i));
+                    "realtime primary-key transport batch field {} does not match declared schema",
+                    i));
             }
         }
-        if (data_batch->field(SpecialFields::kPreparedKeyValueValueKindIndex)->null_count() != 0 ||
-            data_batch->field(SpecialFields::kPreparedKeyValueSequenceNumberIndex)->null_count() !=
-                0 ||
-            data_batch->field(SpecialFields::kPreparedKeyValueRealtimeOffsetIndex)->null_count() !=
-                0) {
-            return Status::Invalid("prepared transport columns must not contain nulls");
+        if (data_batch->field(RealtimePrimaryKeyLayout::kValueKindIndex)->null_count() != 0 ||
+            data_batch->field(RealtimePrimaryKeyLayout::kSequenceNumberIndex)->null_count() != 0 ||
+            data_batch->field(RealtimePrimaryKeyLayout::kRealtimeOffsetIndex)->null_count() != 0) {
+            return Status::Invalid("realtime primary-key transport columns must not contain nulls");
         }
         return Status::OK();
     }
 
     Result<bool> SelectRows(const arrow::Int64Array& offsets, RoaringBitmap32&& selection) {
         for (auto iter = selection.Begin(); iter != selection.End(); ++iter) {
-            const int32_t row = *iter;
-            if (row < 0 || row >= offsets.length()) {
+            const uint32_t row = *iter;
+            if (static_cast<int64_t>(row) >= offsets.length()) {
                 return Status::Invalid(
-                    fmt::format("selected row id {} is out of bounds for prepared batch length {}",
+                    fmt::format("selected row id {} is out of bounds for realtime primary-key "
+                                "transport batch length {}",
                                 row, offsets.length()));
             }
         }
-        if (visible_offsets_.has_value() && selection.Cardinality() != offsets.length()) {
+        if (selection.Cardinality() != offsets.length()) {
             return Status::Invalid(
-                "PK real-time store query reader bitmap must cover every raw mutation");
-        }
-        if (!visible_offsets_.has_value()) {
-            selected_rows_.reserve(offsets.length());
-            for (int64_t row = 0; row < offsets.length(); ++row) {
-                selected_rows_.push_back(row);
-            }
-            return true;
+                "PK real-time store reader bitmap must cover every raw "
+                "transport row");
         }
-        for (auto iter = selection.Begin(); iter != selection.End(); ++iter) {
-            const int32_t row = *iter;
-            const int64_t offset = offsets.Value(row);
-            if (offset >= visible_offsets_->begin && offset < visible_offsets_->end) {
+        selected_rows_.reserve(offsets.length());
+        for (int64_t row = 0; row < offsets.length(); ++row) {
+            if (!visible_offsets_.has_value() || (offsets.Value(row) >= visible_offsets_->begin &&
+                                                  offsets.Value(row) < visible_offsets_->end)) {
                 selected_rows_.push_back(row);
             }
         }
@@ -407,7 +399,7 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
 
  private:
     std::unique_ptr<BatchReader> reader_;
-    std::shared_ptr<const PreparedReaderPlan> plan_;
+    std::shared_ptr<const RealtimePrimaryKeyReaderPlan> plan_;
     std::optional<OffsetRange> visible_offsets_;
     std::shared_ptr<MemoryPool> pool_;
     std::shared_ptr<RealtimeOffsetCoverage> offset_coverage_;
@@ -421,31 +413,39 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
 
 }  // namespace
 
-Status PreparedKeyValueReaderFactory::ValidateTransportSchema(
-    const std::shared_ptr<arrow::Schema>& prepared_schema) {
-    if (!prepared_schema ||
-        prepared_schema->num_fields() < SpecialFields::kPreparedKeyValueValueStartIndex) {
-        return Status::Invalid("prepared schema must contain realtime transport fields");
+std::shared_ptr<arrow::Schema> RealtimePrimaryKeyLayout::CreateSchema(
+    const std::vector<std::shared_ptr<arrow::Field>>& value_fields) {
+    arrow::FieldVector fields = {
+        DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
+        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
+            ->WithNullable(false),
+        DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset())};
+    fields.insert(fields.end(), value_fields.begin(), value_fields.end());
+    return arrow::schema(std::move(fields));
+}
+
+Status RealtimePrimaryKeyLayout::ValidateSchema(
+    const std::shared_ptr<arrow::Schema>& transport_schema) {
+    if (!transport_schema || transport_schema->num_fields() < kValueStartIndex) {
+        return Status::Invalid(
+            "realtime primary-key transport schema must contain transport fields");
     }
-    PAIMON_RETURN_NOT_OK(CheckPreparedField(prepared_schema,
-                                            SpecialFields::kPreparedKeyValueValueKindIndex,
-                                            SpecialFields::ValueKind()));
-    PAIMON_RETURN_NOT_OK(CheckPreparedField(prepared_schema,
-                                            SpecialFields::kPreparedKeyValueSequenceNumberIndex,
-                                            SpecialFields::SequenceNumber()));
-    PAIMON_RETURN_NOT_OK(CheckPreparedField(prepared_schema,
-                                            SpecialFields::kPreparedKeyValueRealtimeOffsetIndex,
-                                            SpecialFields::RealtimeOffset()));
+    PAIMON_RETURN_NOT_OK(
+        CheckTransportField(transport_schema, kValueKindIndex, SpecialFields::ValueKind()));
+    PAIMON_RETURN_NOT_OK(CheckTransportField(transport_schema, kSequenceNumberIndex,
+                                             SpecialFields::SequenceNumber()));
+    PAIMON_RETURN_NOT_OK(CheckTransportField(transport_schema, kRealtimeOffsetIndex,
+                                             SpecialFields::RealtimeOffset()));
     return Status::OK();
 }
 
 Result<std::vector<std::unique_ptr<KeyValueRecordReader>>>
-PreparedKeyValueReaderFactory::CreateForQuery(std::vector<std::unique_ptr<BatchReader>>&& readers,
-                                              const std::shared_ptr<arrow::Schema>& prepared_schema,
-                                              const OffsetRange& visible_offsets,
-                                              const std::shared_ptr<arrow::Schema>& key_schema,
-                                              const std::shared_ptr<arrow::Schema>& value_schema,
-                                              const std::shared_ptr<MemoryPool>& memory_pool) {
+RealtimePrimaryKeyReaderFactory::CreateForQuery(
+    std::vector<std::unique_ptr<BatchReader>>&& readers,
+    const std::shared_ptr<arrow::Schema>& transport_schema, const OffsetRange& visible_offsets,
+    const std::shared_ptr<arrow::Schema>& key_schema,
+    const std::shared_ptr<arrow::Schema>& value_schema,
+    const std::shared_ptr<MemoryPool>& memory_pool) {
     std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers;
     ScopeGuard remaining_raw_readers_guard([&readers]() { CloseReaders(readers); });
     if (readers.empty() && visible_offsets.begin < visible_offsets.end) {
@@ -457,15 +457,16 @@ PreparedKeyValueReaderFactory::CreateForQuery(std::vector<std::unique_ptr<BatchR
             return Status::Invalid("PK real-time store returned a null query reader");
         }
     }
-    PAIMON_RETURN_NOT_OK(ValidateTransportSchema(prepared_schema));
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<const PreparedReaderPlan> plan,
-                           PreparedReaderPlan::Create(prepared_schema, key_schema, value_schema));
+    PAIMON_RETURN_NOT_OK(RealtimePrimaryKeyLayout::ValidateSchema(transport_schema));
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<const RealtimePrimaryKeyReaderPlan> plan,
+        RealtimePrimaryKeyReaderPlan::Create(transport_schema, key_schema, value_schema));
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeOffsetCoverage> offset_coverage,
                            RealtimeOffsetCoverage::Create(visible_offsets, readers.size(),
                                                           /*allow_committed_prefix=*/true));
     adapted_readers.reserve(readers.size());
     for (std::unique_ptr<BatchReader>& reader : readers) {
-        adapted_readers.push_back(std::make_unique<PreparedKeyValueReader>(
+        adapted_readers.push_back(std::make_unique<RealtimePrimaryKeyReader>(
             std::move(reader), plan, visible_offsets, memory_pool, offset_coverage));
     }
     remaining_raw_readers_guard.Release();
@@ -473,9 +474,9 @@ PreparedKeyValueReaderFactory::CreateForQuery(std::vector<std::unique_ptr<BatchR
 }
 
 Result<std::vector<std::unique_ptr<KeyValueRecordReader>>>
-PreparedKeyValueReaderFactory::CreateForCommit(
+RealtimePrimaryKeyReaderFactory::CreateForCommit(
     std::vector<std::unique_ptr<BatchReader>>&& readers,
-    const std::shared_ptr<arrow::Schema>& prepared_schema, const OffsetRange& sealed_offsets,
+    const std::shared_ptr<arrow::Schema>& transport_schema, const OffsetRange& sealed_offsets,
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
     const std::shared_ptr<MemoryPool>& memory_pool) {
@@ -490,15 +491,16 @@ PreparedKeyValueReaderFactory::CreateForCommit(
             return Status::Invalid("PK real-time store returned a null commit reader");
         }
     }
-    PAIMON_RETURN_NOT_OK(ValidateTransportSchema(prepared_schema));
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<const PreparedReaderPlan> plan,
-                           PreparedReaderPlan::Create(prepared_schema, key_schema, value_schema));
+    PAIMON_RETURN_NOT_OK(RealtimePrimaryKeyLayout::ValidateSchema(transport_schema));
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<const RealtimePrimaryKeyReaderPlan> plan,
+        RealtimePrimaryKeyReaderPlan::Create(transport_schema, key_schema, value_schema));
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeOffsetCoverage> offset_coverage,
                            RealtimeOffsetCoverage::Create(sealed_offsets, readers.size(),
                                                           /*allow_committed_prefix=*/false));
     adapted_readers.reserve(readers.size());
     for (std::unique_ptr<BatchReader>& reader : readers) {
-        adapted_readers.push_back(std::make_unique<PreparedKeyValueReader>(
+        adapted_readers.push_back(std::make_unique<RealtimePrimaryKeyReader>(
             std::move(reader), plan, std::nullopt, memory_pool, offset_coverage));
     }
     remaining_raw_readers_guard.Release();
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.h b/src/paimon/core/realtime/realtime_primary_key_reader.h
similarity index 63%
rename from src/paimon/core/realtime/prepared_key_value_reader.h
rename to src/paimon/core/realtime/realtime_primary_key_reader.h
index bb03e2ad4..d175c3b63 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.h
+++ b/src/paimon/core/realtime/realtime_primary_key_reader.h
@@ -18,6 +18,7 @@
 
 #pragma once
 
+#include <cstdint>
 #include <memory>
 #include <vector>
 
@@ -30,23 +31,38 @@ namespace paimon {
 class BatchReader;
 class MemoryPool;
 
-class PreparedKeyValueReaderFactory {
+/// Defines the Arrow field layout for PK realtime transport batches.
+class RealtimePrimaryKeyLayout {
  public:
-    PreparedKeyValueReaderFactory() = delete;
-    ~PreparedKeyValueReaderFactory() = delete;
+    RealtimePrimaryKeyLayout() = delete;
+    ~RealtimePrimaryKeyLayout() = delete;
 
-    static Status ValidateTransportSchema(const std::shared_ptr<arrow::Schema>& prepared_schema);
+    static constexpr int32_t kValueKindIndex = 0;
+    static constexpr int32_t kSequenceNumberIndex = 1;
+    static constexpr int32_t kRealtimeOffsetIndex = 2;
+    static constexpr int32_t kValueStartIndex = 3;
+
+    static std::shared_ptr<arrow::Schema> CreateSchema(
+        const std::vector<std::shared_ptr<arrow::Field>>& value_fields);
+
+    static Status ValidateSchema(const std::shared_ptr<arrow::Schema>& transport_schema);
+};
+
+class RealtimePrimaryKeyReaderFactory {
+ public:
+    RealtimePrimaryKeyReaderFactory() = delete;
+    ~RealtimePrimaryKeyReaderFactory() = delete;
 
     static Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateForQuery(
         std::vector<std::unique_ptr<BatchReader>>&& readers,
-        const std::shared_ptr<arrow::Schema>& prepared_schema, const OffsetRange& visible_offsets,
+        const std::shared_ptr<arrow::Schema>& transport_schema, const OffsetRange& visible_offsets,
         const std::shared_ptr<arrow::Schema>& key_schema,
         const std::shared_ptr<arrow::Schema>& value_schema,
         const std::shared_ptr<MemoryPool>& memory_pool);
 
     static Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateForCommit(
         std::vector<std::unique_ptr<BatchReader>>&& readers,
-        const std::shared_ptr<arrow::Schema>& prepared_schema, const OffsetRange& sealed_offsets,
+        const std::shared_ptr<arrow::Schema>& transport_schema, const OffsetRange& sealed_offsets,
         const std::shared_ptr<arrow::Schema>& key_schema,
         const std::shared_ptr<arrow::Schema>& value_schema,
         const std::shared_ptr<MemoryPool>& memory_pool);
diff --git a/src/paimon/core/realtime/prepared_key_value_reader_test.cpp b/src/paimon/core/realtime/realtime_primary_key_reader_test.cpp
similarity index 56%
rename from src/paimon/core/realtime/prepared_key_value_reader_test.cpp
rename to src/paimon/core/realtime/realtime_primary_key_reader_test.cpp
index c0515df72..beb1a6435 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader_test.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_reader_test.cpp
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-#include "paimon/core/realtime/prepared_key_value_reader.h"
+#include "paimon/core/realtime/realtime_primary_key_reader.h"
 
 #include <cstdint>
 #include <memory>
@@ -49,40 +49,34 @@ std::shared_ptr<arrow::Field> MakeField(const std::string& name,
         DataField(field_id, arrow::field(name, type, nullable)));
 }
 
-std::shared_ptr<arrow::Schema> MakePreparedSchema(const arrow::FieldVector& value_fields) {
-    arrow::FieldVector prepared_fields = {
-        DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
-        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
-            ->WithNullable(false),
-        DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset())};
-    prepared_fields.insert(prepared_fields.end(), value_fields.begin(), value_fields.end());
-    return arrow::schema(prepared_fields);
+std::shared_ptr<arrow::Schema> MakeTransportSchema(const arrow::FieldVector& value_fields) {
+    return RealtimePrimaryKeyLayout::CreateSchema(value_fields);
 }
 
-Result<std::unique_ptr<KeyValueRecordReader>> CreatePreparedQueryReaderForTest(
-    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
+Result<std::unique_ptr<KeyValueRecordReader>> CreateRealtimePrimaryKeyQueryReaderForTest(
+    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& transport_schema,
     const OffsetRange& visible_offsets, const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
     const std::shared_ptr<MemoryPool>& memory_pool) {
     std::vector<std::unique_ptr<BatchReader>> readers;
     readers.push_back(std::move(reader));
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers,
-                           PreparedKeyValueReaderFactory::CreateForQuery(
-                               std::move(readers), prepared_schema, visible_offsets, key_schema,
+                           RealtimePrimaryKeyReaderFactory::CreateForQuery(
+                               std::move(readers), transport_schema, visible_offsets, key_schema,
                                value_schema, memory_pool));
     return std::move(adapted_readers[0]);
 }
 
-Result<std::unique_ptr<KeyValueRecordReader>> CreatePreparedCommitReaderForTest(
-    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
+Result<std::unique_ptr<KeyValueRecordReader>> CreateRealtimePrimaryKeyCommitReaderForTest(
+    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& transport_schema,
     const OffsetRange& sealed_offsets, const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
     const std::shared_ptr<MemoryPool>& memory_pool) {
     std::vector<std::unique_ptr<BatchReader>> readers;
     readers.push_back(std::move(reader));
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers,
-                           PreparedKeyValueReaderFactory::CreateForCommit(
-                               std::move(readers), prepared_schema, sealed_offsets, key_schema,
+                           RealtimePrimaryKeyReaderFactory::CreateForCommit(
+                               std::move(readers), transport_schema, sealed_offsets, key_schema,
                                value_schema, memory_pool));
     return std::move(adapted_readers[0]);
 }
@@ -142,21 +136,69 @@ class MalformedBitmapBatchReader : public BatchReader {
 
 }  // namespace
 
-class PreparedKeyValueReaderTest : public testing::Test {
+class RealtimePrimaryKeyReaderTest : public testing::Test {
  protected:
     std::shared_ptr<MemoryPool> pool_ = GetDefaultPool();
 };
 
-TEST_F(PreparedKeyValueReaderTest, TestQueryAllowsCommittedPrefix) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestTransportSchemaLayout) {
+    arrow::FieldVector value_fields = {arrow::field("key", arrow::int64(), false),
+                                       arrow::field("value", arrow::utf8())};
+    std::shared_ptr<arrow::Schema> schema = MakeTransportSchema(value_fields);
+
+    ASSERT_EQ(RealtimePrimaryKeyLayout::kValueKindIndex, 0);
+    ASSERT_EQ(RealtimePrimaryKeyLayout::kSequenceNumberIndex, 1);
+    ASSERT_EQ(RealtimePrimaryKeyLayout::kRealtimeOffsetIndex, 2);
+    ASSERT_EQ(RealtimePrimaryKeyLayout::kValueStartIndex, 3);
+    ASSERT_EQ(schema->field(0)->name(), "_VALUE_KIND");
+    ASSERT_EQ(schema->field(1)->name(), "_SEQUENCE_NUMBER");
+    ASSERT_EQ(schema->field(2)->name(), "_REALTIME_OFFSET");
+    ASSERT_EQ(schema->field(3)->name(), "key");
+    ASSERT_EQ(schema->field(4)->name(), "value");
+    ASSERT_FALSE(schema->field(0)->nullable());
+    ASSERT_FALSE(schema->field(1)->nullable());
+    ASSERT_EQ(schema->field(2)->nullable(), SpecialFields::RealtimeOffset().Nullable());
+    ASSERT_FALSE(schema->field(3)->nullable());
+    ASSERT_TRUE(schema->field(4)->nullable());
+}
+
+TEST_F(RealtimePrimaryKeyReaderTest, TestTransportSchemaValidation) {
+    const std::shared_ptr<arrow::Schema> valid = MakeTransportSchema({});
+    std::vector<arrow::FieldVector> invalid_fields;
+
+    arrow::FieldVector wrong_type = valid->fields();
+    wrong_type[0] = DataField::ConvertDataFieldToArrowField(
+                        DataField(SpecialFields::ValueKind().Id(),
+                                  arrow::field("_VALUE_KIND", arrow::int32(), false)))
+                        ->WithNullable(false);
+    invalid_fields.push_back(std::move(wrong_type));
+
+    arrow::FieldVector nullable_sequence = valid->fields();
+    nullable_sequence[1] = nullable_sequence[1]->WithNullable(true);
+    invalid_fields.push_back(std::move(nullable_sequence));
+
+    arrow::FieldVector wrong_offset_id = valid->fields();
+    wrong_offset_id[2] = DataField::ConvertDataFieldToArrowField(
+                             DataField(99, arrow::field("_REALTIME_OFFSET", arrow::int64(), false)))
+                             ->WithNullable(false);
+    invalid_fields.push_back(std::move(wrong_offset_id));
+
+    for (const arrow::FieldVector& fields : invalid_fields) {
+        ASSERT_NOK_WITH_MSG(RealtimePrimaryKeyLayout::ValidateSchema(arrow::schema(fields)),
+                            "transport schema field");
+    }
+}
+
+TEST_F(RealtimePrimaryKeyReaderTest, TestQueryAllowsCommittedPrefix) {
     std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32())),
                                            DataField(1, arrow::field("v0", arrow::int32()))};
     std::shared_ptr<arrow::Schema> value_schema =
         DataField::ConvertDataFieldsToArrowSchema(value_fields);
     std::shared_ptr<arrow::Schema> key_schema = arrow::schema({value_schema->field(0)});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema(value_schema->fields());
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema(value_schema->fields());
+    std::shared_ptr<arrow::DataType> transport_type = arrow::struct_(transport_schema->fields());
+    auto transport_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(transport_type, R"([
         [0, 100, 0, 1, 10],
         [0, 101, 1, 2, 20],
         [0, 102, 2, 4, 40],
@@ -166,10 +208,10 @@ TEST_F(PreparedKeyValueReaderTest, TestQueryAllowsCommittedPrefix) {
 
     std::vector<std::unique_ptr<BatchReader>> batch_readers;
     batch_readers.push_back(
-        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 2));
+        std::make_unique<MockFileBatchReader>(transport_array, transport_type, 2));
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
-                         PreparedKeyValueReaderFactory::CreateForQuery(
-                             std::move(batch_readers), prepared_schema, OffsetRange(2, 4),
+                         RealtimePrimaryKeyReaderFactory::CreateForQuery(
+                             std::move(batch_readers), transport_schema, OffsetRange(2, 4),
                              key_schema, value_schema, pool_));
     ASSERT_EQ(1, readers.size());
     ASSERT_OK_AND_ASSIGN(
@@ -185,46 +227,46 @@ TEST_F(PreparedKeyValueReaderTest, TestQueryAllowsCommittedPrefix) {
     KeyValueChecker::CheckResult(expected, results, 1, 2);
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestQueryRejectsNegativeOffset) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestQueryRejectsNegativeOffset) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, -1, 1]])")
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema({key});
+    std::shared_ptr<arrow::DataType> transport_type = arrow::struct_(transport_schema->fields());
+    std::shared_ptr<arrow::Array> transport_array =
+        arrow::ipc::internal::json::ArrayFromJSON(transport_type, R"([[0, 10, -1, 1]])")
             .ValueOrDie();
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> reader,
-        CreatePreparedQueryReaderForTest(
-            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type,
+        CreateRealtimePrimaryKeyQueryReaderForTest(
+            std::make_unique<MockFileBatchReader>(transport_array, transport_type,
                                                   /*read_batch_size=*/1),
-            prepared_schema, OffsetRange(1, 2), value_schema, value_schema, pool_));
+            transport_schema, OffsetRange(1, 2), value_schema, value_schema, pool_));
 
     ASSERT_NOK_WITH_MSG(reader->NextBatch(), "reader offset must be non-negative");
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestQueryOffsetCoverageAcrossReadersAndBatches) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestQueryOffsetCoverageAcrossReadersAndBatches) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema({key});
+    std::shared_ptr<arrow::DataType> transport_type = arrow::struct_(transport_schema->fields());
     std::shared_ptr<arrow::Array> first_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+        arrow::ipc::internal::json::ArrayFromJSON(transport_type,
                                                   R"([[0, 10, 2, 1], [0, 11, 0, 2]])")
             .ValueOrDie();
     std::shared_ptr<arrow::Array> second_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+        arrow::ipc::internal::json::ArrayFromJSON(transport_type,
                                                   R"([[0, 12, 3, 3], [0, 13, 1, 4]])")
             .ValueOrDie();
     std::vector<std::unique_ptr<BatchReader>> batch_readers;
     batch_readers.push_back(
-        std::make_unique<MockFileBatchReader>(first_array, prepared_type, /*read_batch_size=*/1));
+        std::make_unique<MockFileBatchReader>(first_array, transport_type, /*read_batch_size=*/1));
     batch_readers.push_back(
-        std::make_unique<MockFileBatchReader>(second_array, prepared_type, /*read_batch_size=*/1));
+        std::make_unique<MockFileBatchReader>(second_array, transport_type, /*read_batch_size=*/1));
 
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
-                         PreparedKeyValueReaderFactory::CreateForQuery(
-                             std::move(batch_readers), prepared_schema, OffsetRange(0, 4),
+                         RealtimePrimaryKeyReaderFactory::CreateForQuery(
+                             std::move(batch_readers), transport_schema, OffsetRange(0, 4),
                              value_schema, value_schema, pool_));
     int64_t row_count = 0;
     for (const std::unique_ptr<KeyValueRecordReader>& reader : readers) {
@@ -237,21 +279,21 @@ TEST_F(PreparedKeyValueReaderTest, TestQueryOffsetCoverageAcrossReadersAndBatche
     ASSERT_EQ(4, row_count);
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestQueryRejectsMissingVisibleOffset) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestQueryRejectsMissingVisibleOffset) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema({key});
+    std::shared_ptr<arrow::DataType> transport_type = arrow::struct_(transport_schema->fields());
+    std::shared_ptr<arrow::Array> transport_array =
+        arrow::ipc::internal::json::ArrayFromJSON(transport_type,
                                                   R"([[0, 10, 0, 1], [0, 11, 2, 2]])")
             .ValueOrDie();
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> reader,
-        CreatePreparedQueryReaderForTest(
-            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type,
+        CreateRealtimePrimaryKeyQueryReaderForTest(
+            std::make_unique<MockFileBatchReader>(transport_array, transport_type,
                                                   /*read_batch_size=*/1),
-            prepared_schema, OffsetRange(0, 3), value_schema, value_schema, pool_));
+            transport_schema, OffsetRange(0, 3), value_schema, value_schema, pool_));
 
     ASSERT_NOK_WITH_MSG(
         (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
@@ -259,25 +301,26 @@ TEST_F(PreparedKeyValueReaderTest, TestQueryRejectsMissingVisibleOffset) {
         "query readers did not cover the visible range");
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestQueryRejectsDuplicateVisibleOffset) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestQueryRejectsDuplicateVisibleOffset) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema({key});
+    std::shared_ptr<arrow::DataType> transport_type = arrow::struct_(transport_schema->fields());
     std::shared_ptr<arrow::Array> first_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1]])").ValueOrDie();
+        arrow::ipc::internal::json::ArrayFromJSON(transport_type, R"([[0, 10, 0, 1]])")
+            .ValueOrDie();
     std::shared_ptr<arrow::Array> second_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+        arrow::ipc::internal::json::ArrayFromJSON(transport_type,
                                                   R"([[0, 11, 1, 2], [0, 12, 1, 3]])")
             .ValueOrDie();
     std::vector<std::unique_ptr<BatchReader>> batch_readers;
     batch_readers.push_back(
-        std::make_unique<MockFileBatchReader>(first_array, prepared_type, /*read_batch_size=*/1));
+        std::make_unique<MockFileBatchReader>(first_array, transport_type, /*read_batch_size=*/1));
     batch_readers.push_back(
-        std::make_unique<MockFileBatchReader>(second_array, prepared_type, /*read_batch_size=*/1));
+        std::make_unique<MockFileBatchReader>(second_array, transport_type, /*read_batch_size=*/1));
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
-                         PreparedKeyValueReaderFactory::CreateForQuery(
-                             std::move(batch_readers), prepared_schema, OffsetRange(0, 2),
+                         RealtimePrimaryKeyReaderFactory::CreateForQuery(
+                             std::move(batch_readers), transport_schema, OffsetRange(0, 2),
                              value_schema, value_schema, pool_));
     ASSERT_OK_AND_ASSIGN(
         std::vector<KeyValue> first_rows,
@@ -290,109 +333,134 @@ TEST_F(PreparedKeyValueReaderTest, TestQueryRejectsDuplicateVisibleOffset) {
                         "query readers did not cover the visible range");
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestQueryRejectsEmptyEofForVisibleRange) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestQueryRejectsEmptyEofForVisibleRange) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([])").ValueOrDie();
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema({key});
+    std::shared_ptr<arrow::DataType> transport_type = arrow::struct_(transport_schema->fields());
+    std::shared_ptr<arrow::Array> transport_array =
+        arrow::ipc::internal::json::ArrayFromJSON(transport_type, R"([])").ValueOrDie();
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> reader,
-        CreatePreparedQueryReaderForTest(
-            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type,
+        CreateRealtimePrimaryKeyQueryReaderForTest(
+            std::make_unique<MockFileBatchReader>(transport_array, transport_type,
                                                   /*read_batch_size=*/1),
-            prepared_schema, OffsetRange(0, 1), value_schema, value_schema, pool_));
+            transport_schema, OffsetRange(0, 1), value_schema, value_schema, pool_));
 
     ASSERT_NOK_WITH_MSG(reader->NextBatch(), "query readers did not cover the visible range");
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestQueryRejectsEmptyReadersForVisibleRange) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestQueryRejectsEmptyReadersForVisibleRange) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema({key});
     std::vector<std::unique_ptr<BatchReader>> batch_readers;
 
     ASSERT_NOK_WITH_MSG(
-        PreparedKeyValueReaderFactory::CreateForQuery(std::move(batch_readers), prepared_schema,
-                                                      OffsetRange(0, 1), value_schema, value_schema,
-                                                      pool_),
+        RealtimePrimaryKeyReaderFactory::CreateForQuery(std::move(batch_readers), transport_schema,
+                                                        OffsetRange(0, 1), value_schema,
+                                                        value_schema, pool_),
         "PK real-time store returned no query readers for a non-empty visible range");
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestQueryAllowsEmptyReadersForEmptyVisibleRange) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestQueryAllowsEmptyReadersForEmptyVisibleRange) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema({key});
     std::vector<std::unique_ptr<BatchReader>> batch_readers;
 
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
-                         PreparedKeyValueReaderFactory::CreateForQuery(
-                             std::move(batch_readers), prepared_schema, OffsetRange(1, 1),
+                         RealtimePrimaryKeyReaderFactory::CreateForQuery(
+                             std::move(batch_readers), transport_schema, OffsetRange(1, 1),
                              value_schema, value_schema, pool_));
     ASSERT_TRUE(readers.empty());
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestPreparedReaderBitmapBounds) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestQueryBitmapBounds) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1]])").ValueOrDie();
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema({key});
+    std::shared_ptr<arrow::DataType> transport_type = arrow::struct_(transport_schema->fields());
+    std::shared_ptr<arrow::Array> transport_array =
+        arrow::ipc::internal::json::ArrayFromJSON(transport_type, R"([[0, 10, 0, 1]])")
+            .ValueOrDie();
     auto batch_reader = std::make_unique<MalformedBitmapBatchReader>(
-        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, /*batch_size=*/1),
+        std::make_unique<MockFileBatchReader>(transport_array, transport_type, /*batch_size=*/1),
         /*row_id=*/1);
 
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
-                                         OffsetRange(0, 1), value_schema, value_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
+                         CreateRealtimePrimaryKeyQueryReaderForTest(
+                             std::move(batch_reader), transport_schema, OffsetRange(0, 1),
+                             value_schema, value_schema, pool_));
     Result<std::vector<KeyValue>> result =
         ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
                                                    KeyValueRecordReader::Iterator>(reader.get());
     ASSERT_TRUE(result.status().IsInvalid());
-    ASSERT_NOK_WITH_MSG(result, "selected row id 1 is out of bounds for prepared batch length 1");
+    ASSERT_NOK_WITH_MSG(result,
+                        "selected row id 1 is out of bounds for realtime primary-key transport "
+                        "batch length 1");
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestPreparedQueryReaderRejectsPartialBitmap) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestQueryRejectsPartialBitmap) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema({key});
+    std::shared_ptr<arrow::DataType> transport_type = arrow::struct_(transport_schema->fields());
+    std::shared_ptr<arrow::Array> transport_array =
+        arrow::ipc::internal::json::ArrayFromJSON(transport_type,
                                                   R"([[0, 10, 0, 1], [0, 11, 1, 2]])")
             .ValueOrDie();
     RoaringBitmap32 partial_bitmap;
     partial_bitmap.Add(0);
     auto batch_reader = std::make_unique<MockFileBatchReader>(
-        prepared_array, prepared_type, partial_bitmap, /*read_batch_size=*/2);
+        transport_array, transport_type, partial_bitmap, /*read_batch_size=*/2);
     batch_reader->EnableRandomizeBatchSize(false);
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
-                                         OffsetRange(0, 2), value_schema, value_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
+                         CreateRealtimePrimaryKeyQueryReaderForTest(
+                             std::move(batch_reader), transport_schema, OffsetRange(0, 2),
+                             value_schema, value_schema, pool_));
 
-    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "must cover every raw mutation");
+    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "must cover every raw transport row");
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestPreparedReaderQueryProjection) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestCommitRejectsPartialBitmap) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema({key});
+    std::shared_ptr<arrow::DataType> transport_type = arrow::struct_(transport_schema->fields());
+    std::shared_ptr<arrow::Array> transport_array =
+        arrow::ipc::internal::json::ArrayFromJSON(transport_type,
+                                                  R"([[0, 10, 0, 1], [0, 11, 1, 2]])")
+            .ValueOrDie();
+    RoaringBitmap32 partial_bitmap;
+    partial_bitmap.Add(0);
+    auto batch_reader = std::make_unique<MockFileBatchReader>(
+        transport_array, transport_type, partial_bitmap, /*read_batch_size=*/2);
+    batch_reader->EnableRandomizeBatchSize(false);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
+                         CreateRealtimePrimaryKeyCommitReaderForTest(
+                             std::move(batch_reader), transport_schema, OffsetRange(0, 2),
+                             value_schema, value_schema, pool_));
+
+    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "must cover every raw transport row");
+}
+
+TEST_F(RealtimePrimaryKeyReaderTest, TestQueryProjection) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Field> extra = MakeField("extra", arrow::int32(), 1);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key, extra});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1, 2]])")
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema({key, extra});
+    std::shared_ptr<arrow::DataType> transport_type = arrow::struct_(transport_schema->fields());
+    auto transport_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(transport_type, R"([[0, 10, 0, 1, 2]])")
             .ValueOrDie());
 
     auto query_batch_reader =
-        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> query_reader,
-        CreatePreparedQueryReaderForTest(std::move(query_batch_reader), prepared_schema,
-                                         OffsetRange(0, 1), value_schema, value_schema, pool_));
+        std::make_unique<MockFileBatchReader>(transport_array, transport_type, 1);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> query_reader,
+                         CreateRealtimePrimaryKeyQueryReaderForTest(
+                             std::move(query_batch_reader), transport_schema, OffsetRange(0, 1),
+                             value_schema, value_schema, pool_));
     ASSERT_OK_AND_ASSIGN(
         std::vector<KeyValue> query_results,
         (ReadResultCollector::CollectKeyValueResult<
@@ -402,28 +470,28 @@ TEST_F(PreparedKeyValueReaderTest, TestPreparedReaderQueryProjection) {
     ASSERT_EQ(query_results[0].value->GetInt(0), 1);
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestCommitOffsetCoverage) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestCommitOffsetCoverage) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema({key});
+    std::shared_ptr<arrow::DataType> transport_type = arrow::struct_(transport_schema->fields());
     std::shared_ptr<arrow::Array> first_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+        arrow::ipc::internal::json::ArrayFromJSON(transport_type,
                                                   R"([[0, 10, 2, 1], [0, 11, 0, 3]])")
             .ValueOrDie();
     std::shared_ptr<arrow::Array> second_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+        arrow::ipc::internal::json::ArrayFromJSON(transport_type,
                                                   R"([[0, 12, 1, 2], [0, 13, 3, 4]])")
             .ValueOrDie();
     std::vector<std::unique_ptr<BatchReader>> batch_readers;
     batch_readers.push_back(
-        std::make_unique<MockFileBatchReader>(first_array, prepared_type, /*read_batch_size=*/1));
+        std::make_unique<MockFileBatchReader>(first_array, transport_type, /*read_batch_size=*/1));
     batch_readers.push_back(
-        std::make_unique<MockFileBatchReader>(second_array, prepared_type, /*read_batch_size=*/1));
+        std::make_unique<MockFileBatchReader>(second_array, transport_type, /*read_batch_size=*/1));
 
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
-                         PreparedKeyValueReaderFactory::CreateForCommit(
-                             std::move(batch_readers), prepared_schema, OffsetRange(0, 4),
+                         RealtimePrimaryKeyReaderFactory::CreateForCommit(
+                             std::move(batch_readers), transport_schema, OffsetRange(0, 4),
                              value_schema, value_schema, pool_));
     int64_t row_count = 0;
     for (const std::unique_ptr<KeyValueRecordReader>& reader : readers) {
@@ -436,34 +504,34 @@ TEST_F(PreparedKeyValueReaderTest, TestCommitOffsetCoverage) {
     ASSERT_EQ(4, row_count);
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestCommitRejectsEmptyReaders) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestCommitRejectsEmptyReaders) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema({key});
     std::vector<std::unique_ptr<BatchReader>> batch_readers;
 
-    ASSERT_NOK_WITH_MSG(PreparedKeyValueReaderFactory::CreateForCommit(
-                            std::move(batch_readers), prepared_schema, OffsetRange(0, 1),
+    ASSERT_NOK_WITH_MSG(RealtimePrimaryKeyReaderFactory::CreateForCommit(
+                            std::move(batch_readers), transport_schema, OffsetRange(0, 1),
                             value_schema, value_schema, pool_),
                         "PK real-time store returned no commit readers for a sealed segment");
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestRejectsDuplicateCommitOffset) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestRejectsDuplicateCommitOffset) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema({key});
+    std::shared_ptr<arrow::DataType> transport_type = arrow::struct_(transport_schema->fields());
+    std::shared_ptr<arrow::Array> transport_array =
         arrow::ipc::internal::json::ArrayFromJSON(
-            prepared_type, R"([[0, 10, 0, 1], [0, 11, 0, 2], [0, 12, 2, 3]])")
+            transport_type, R"([[0, 10, 0, 1], [0, 11, 0, 2], [0, 12, 2, 3]])")
             .ValueOrDie();
     std::vector<std::unique_ptr<BatchReader>> batch_readers;
-    batch_readers.push_back(std::make_unique<MockFileBatchReader>(prepared_array, prepared_type,
+    batch_readers.push_back(std::make_unique<MockFileBatchReader>(transport_array, transport_type,
                                                                   /*read_batch_size=*/1));
 
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
-                         PreparedKeyValueReaderFactory::CreateForCommit(
-                             std::move(batch_readers), prepared_schema, OffsetRange(0, 3),
+                         RealtimePrimaryKeyReaderFactory::CreateForCommit(
+                             std::move(batch_readers), transport_schema, OffsetRange(0, 3),
                              value_schema, value_schema, pool_));
     ASSERT_NOK_WITH_MSG((ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
                                                                     KeyValueRecordReader::Iterator>(
@@ -471,30 +539,30 @@ TEST_F(PreparedKeyValueReaderTest, TestRejectsDuplicateCommitOffset) {
                         "did not cover the sealed range");
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestBadCommitBatch) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestBadCommitBatch) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Field> value = MakeField("value", arrow::int32(), 1);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key, value});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key, value});
-    std::shared_ptr<arrow::Schema> actual_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema({key, value});
+    std::shared_ptr<arrow::Schema> actual_schema = MakeTransportSchema({key});
     std::shared_ptr<arrow::DataType> actual_type = arrow::struct_(actual_schema->fields());
     std::shared_ptr<arrow::Array> actual =
         arrow::ipc::internal::json::ArrayFromJSON(actual_type, R"([[0, 10, 0, 1]])").ValueOrDie();
 
     auto batch_reader = std::make_unique<MockFileBatchReader>(actual, actual_type, 1);
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
-                         CreatePreparedCommitReaderForTest(std::move(batch_reader), prepared_schema,
-                                                           OffsetRange(0, 1), arrow::schema({key}),
-                                                           value_schema, pool_));
+                         CreateRealtimePrimaryKeyCommitReaderForTest(
+                             std::move(batch_reader), transport_schema, OffsetRange(0, 1),
+                             arrow::schema({key}), value_schema, pool_));
     ASSERT_NOK_WITH_MSG(reader->NextBatch(), "field count");
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestPreparedReaderSafeDecode) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestSafeDecode) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema({key});
 
-    arrow::FieldVector invalid_fields = prepared_schema->fields();
+    arrow::FieldVector invalid_fields = transport_schema->fields();
     invalid_fields[0] = invalid_fields[0]->WithName("wrong_value_kind");
     invalid_fields[3] = MakeField("wrong_key", arrow::int32(), 99);
     std::shared_ptr<arrow::DataType> invalid_type = arrow::struct_(invalid_fields);
@@ -502,17 +570,17 @@ TEST_F(PreparedKeyValueReaderTest, TestPreparedReaderSafeDecode) {
         arrow::ipc::internal::json::ArrayFromJSON(invalid_type, R"([[0, 10, 0, 1]])").ValueOrDie());
 
     auto batch_reader = std::make_unique<MockFileBatchReader>(invalid_array, invalid_type, 1);
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
-                                         OffsetRange(0, 1), value_schema, value_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
+                         CreateRealtimePrimaryKeyQueryReaderForTest(
+                             std::move(batch_reader), transport_schema, OffsetRange(0, 1),
+                             value_schema, value_schema, pool_));
     ASSERT_NOK_WITH_MSG(
         (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
                                                     KeyValueRecordReader::Iterator>(reader.get())),
-        "prepared batch field");
+        "transport batch field");
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestPreparedReaderNestedValues) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestNestedValues) {
     std::shared_ptr<arrow::Field> id = MakeField("id", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> key_schema = arrow::schema({id});
     std::shared_ptr<arrow::Field> query_item_b = MakeField("renamed_b", arrow::int32(), 11);
@@ -532,20 +600,20 @@ TEST_F(PreparedKeyValueReaderTest, TestPreparedReaderNestedValues) {
                   arrow::map(arrow::struct_({query_key_right, query_key_left}), arrow::int32()), 4);
     std::shared_ptr<arrow::Schema> query_value_schema =
         arrow::schema({id, query_items, query_attrs, query_keyed_values});
-    std::shared_ptr<arrow::Schema> prepared_schema =
-        MakePreparedSchema(query_value_schema->fields());
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
+    std::shared_ptr<arrow::Schema> transport_schema =
+        MakeTransportSchema(query_value_schema->fields());
+    std::shared_ptr<arrow::DataType> transport_type = arrow::struct_(transport_schema->fields());
+    std::shared_ptr<arrow::Array> transport_array =
         arrow::ipc::internal::json::ArrayFromJSON(
-            prepared_type,
+            transport_type,
             R"([[0, 10, 0, 1, [[200, 100], [400, 300]], [["k1", [8, 7]], ["k2", [10, 9]]], [[[12, 11], 13], [[22, 21], 23]]]])")
             .ValueOrDie();
 
-    auto batch_reader = std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
-                                         OffsetRange(0, 1), key_schema, query_value_schema, pool_));
+    auto batch_reader = std::make_unique<MockFileBatchReader>(transport_array, transport_type, 1);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
+                         CreateRealtimePrimaryKeyQueryReaderForTest(
+                             std::move(batch_reader), transport_schema, OffsetRange(0, 1),
+                             key_schema, query_value_schema, pool_));
     ASSERT_OK_AND_ASSIGN(
         std::vector<KeyValue> results,
         (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
@@ -591,16 +659,16 @@ TEST_F(PreparedKeyValueReaderTest, TestPreparedReaderNestedValues) {
     ASSERT_EQ(keyed_value_map->ValueArray()->GetInt(1), 23);
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestPreparedReaderFactoryFailureClosesReaders) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestFactoryFailureClosesReaders) {
     std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32())),
                                            DataField(1, arrow::field("v0", arrow::int32()))};
     std::shared_ptr<arrow::Schema> value_schema =
         DataField::ConvertDataFieldsToArrowSchema(value_fields);
     std::shared_ptr<arrow::Schema> key_schema = arrow::schema({value_schema->field(0)});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema(value_schema->fields());
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema(value_schema->fields());
+    std::shared_ptr<arrow::DataType> transport_type = arrow::struct_(transport_schema->fields());
+    auto transport_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(transport_type, R"([
         [0, 10, 0, 1, 100]
     ])")
             .ValueOrDie());
@@ -608,11 +676,11 @@ TEST_F(PreparedKeyValueReaderTest, TestPreparedReaderFactoryFailureClosesReaders
     int32_t factory_failure_close_count = 0;
     std::vector<std::unique_ptr<BatchReader>> batch_readers;
     batch_readers.push_back(std::make_unique<TrackingBatchReader>(
-        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1),
+        std::make_unique<MockFileBatchReader>(transport_array, transport_type, 1),
         &factory_failure_close_count));
     batch_readers.push_back(nullptr);
-    ASSERT_NOK_WITH_MSG(PreparedKeyValueReaderFactory::CreateForQuery(
-                            std::move(batch_readers), prepared_schema, OffsetRange(0, 1),
+    ASSERT_NOK_WITH_MSG(RealtimePrimaryKeyReaderFactory::CreateForQuery(
+                            std::move(batch_readers), transport_schema, OffsetRange(0, 1),
                             key_schema, value_schema, pool_),
                         "PK real-time store returned a null query reader");
     ASSERT_EQ(factory_failure_close_count, 1);
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index 58103c599..61cd14009 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -31,22 +31,24 @@
 #include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
+#include "paimon/core/core_options.h"
 #include "paimon/core/io/merged_key_value_record_reader.h"
-#include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
+#include "paimon/core/mergetree/compact/merge_function.h"
 #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
 #include "paimon/core/mergetree/merge_tree_writer.h"
-#include "paimon/core/realtime/prepared_key_value_reader.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
+#include "paimon/core/realtime/realtime_primary_key_reader.h"
 #include "paimon/core/utils/commit_increment.h"
+#include "paimon/core/utils/primary_key_table_utils.h"
 #include "paimon/macros.h"
 
 namespace paimon {
 
 namespace {
 
-Result<std::shared_ptr<arrow::StructArray>> PrepareBatch(
+Result<std::shared_ptr<arrow::StructArray>> CreateRealtimePrimaryKeyTransportBatch(
     std::unique_ptr<RecordBatch>&& batch, const std::shared_ptr<arrow::Schema>& write_schema,
-    const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::shared_ptr<arrow::Schema>& transport_schema,
     const std::vector<std::string>& trimmed_primary_keys, int64_t first_sequence_number,
     int64_t first_offset, arrow::MemoryPool* arrow_pool) {
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
@@ -81,8 +83,8 @@ Result<std::shared_ptr<arrow::StructArray>> PrepareBatch(
                                   std::move(offset_array)};
     columns.insert(columns.end(), values->fields().begin(), values->fields().end());
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-        std::shared_ptr<arrow::StructArray> prepared,
-        arrow::StructArray::Make(std::move(columns), prepared_schema->fields()));
+        std::shared_ptr<arrow::StructArray> transport,
+        arrow::StructArray::Make(std::move(columns), transport_schema->fields()));
 
     std::vector<arrow::compute::SortKey> sort_keys;
     sort_keys.reserve(trimmed_primary_keys.size() + 1);
@@ -95,10 +97,10 @@ Result<std::shared_ptr<arrow::StructArray>> PrepareBatch(
     arrow::compute::SortOptions options(sort_keys, arrow::compute::NullPlacement::AtStart);
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
         arrow::Datum indices,
-        arrow::compute::SortIndices(arrow::Datum(prepared), options, &context));
+        arrow::compute::SortIndices(arrow::Datum(transport), options, &context));
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
         arrow::Datum sorted,
-        arrow::compute::Take(arrow::Datum(prepared), indices,
+        arrow::compute::Take(arrow::Datum(transport), indices,
                              arrow::compute::TakeOptions::NoBoundsCheck(), &context));
     return checked_pointer_cast<arrow::StructArray>(sorted.make_array());
 }
@@ -108,9 +110,9 @@ Result<std::shared_ptr<arrow::StructArray>> PrepareBatch(
 Result<std::shared_ptr<RealtimePrimaryKeyWriter>> RealtimePrimaryKeyWriter::Create(
     const std::map<std::string, std::string>& partition, int32_t bucket,
     const std::shared_ptr<arrow::Schema>& write_schema,
-    const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::shared_ptr<arrow::Schema>& transport_schema,
     const std::vector<std::string>& trimmed_primary_keys,
-    const std::shared_ptr<FieldsComparator>& key_comparator,
+    const std::shared_ptr<FieldsComparator>& key_comparator, const CoreOptions& options,
     const std::shared_ptr<RealtimeContextImpl>& realtime_context,
     const RealtimeStoreState& store_state, int64_t restored_max_sequence_number,
     const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
@@ -119,6 +121,10 @@ Result<std::shared_ptr<RealtimePrimaryKeyWriter>> RealtimePrimaryKeyWriter::Crea
         restored_max_sequence_number == std::numeric_limits<int64_t>::max()) {
         return Status::Invalid("PK restored sequence number is invalid");
     }
+    if (options.GetMergeEngine() != MergeEngine::DEDUPLICATE) {
+        return Status::NotImplemented("PK realtime supports only the DEDUPLICATE merge engine");
+    }
+    PAIMON_RETURN_NOT_OK(RealtimePrimaryKeyLayout::ValidateSchema(transport_schema));
     arrow::FieldVector key_fields;
     key_fields.reserve(trimmed_primary_keys.size());
     for (const std::string& key : trimmed_primary_keys) {
@@ -134,8 +140,9 @@ Result<std::shared_ptr<RealtimePrimaryKeyWriter>> RealtimePrimaryKeyWriter::Crea
                                partition_bucket, restored_max_sequence_number));
     return std::shared_ptr<RealtimePrimaryKeyWriter>(new RealtimePrimaryKeyWriter(
         store_state.store, merge_tree_writer, realtime_context, partition_bucket, write_schema,
-        prepared_schema, arrow::schema(std::move(key_fields)), trimmed_primary_keys, key_comparator,
-        store_state.initial_offset, initial_max_sequence_number, memory_pool));
+        transport_schema, arrow::schema(std::move(key_fields)), trimmed_primary_keys,
+        key_comparator, options, store_state.initial_offset, initial_max_sequence_number,
+        memory_pool));
 }
 
 RealtimePrimaryKeyWriter::RealtimePrimaryKeyWriter(
@@ -144,11 +151,12 @@ RealtimePrimaryKeyWriter::RealtimePrimaryKeyWriter(
     const std::shared_ptr<RealtimeContextImpl>& realtime_context,
     const RealtimePartitionBucket& partition_bucket,
     const std::shared_ptr<arrow::Schema>& write_schema,
-    const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::shared_ptr<arrow::Schema>& transport_schema,
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::vector<std::string>& trimmed_primary_keys,
-    const std::shared_ptr<FieldsComparator>& key_comparator, int64_t next_offset,
-    int64_t last_sequence_number, const std::shared_ptr<MemoryPool>& memory_pool)
+    const std::shared_ptr<FieldsComparator>& key_comparator, const CoreOptions& options,
+    int64_t next_offset, int64_t last_sequence_number,
+    const std::shared_ptr<MemoryPool>& memory_pool)
     : memory_pool_(memory_pool),
       arrow_pool_(GetArrowPool(memory_pool)),
       realtime_store_(realtime_store),
@@ -156,10 +164,11 @@ RealtimePrimaryKeyWriter::RealtimePrimaryKeyWriter(
       realtime_context_(realtime_context),
       partition_bucket_(partition_bucket),
       write_schema_(write_schema),
-      prepared_schema_(prepared_schema),
+      transport_schema_(transport_schema),
       key_schema_(key_schema),
       trimmed_primary_keys_(trimmed_primary_keys),
       key_comparator_(key_comparator),
+      options_(options),
       next_offset_(next_offset),
       last_sequence_number_(last_sequence_number) {}
 
@@ -190,16 +199,17 @@ Status RealtimePrimaryKeyWriter::Write(std::unique_ptr<RecordBatch>&& batch) {
     }
     const int64_t first_sequence = last_sequence_number_ + 1;
     PAIMON_ASSIGN_OR_RAISE(
-        std::shared_ptr<arrow::StructArray> prepared,
-        PrepareBatch(std::move(batch), write_schema_, prepared_schema_, trimmed_primary_keys_,
-                     first_sequence, next_offset_, arrow_pool_.get()));
+        std::shared_ptr<arrow::StructArray> transport,
+        CreateRealtimePrimaryKeyTransportBatch(std::move(batch), write_schema_, transport_schema_,
+                                               trimmed_primary_keys_, first_sequence, next_offset_,
+                                               arrow_pool_.get()));
     auto output = std::make_unique<ArrowArray>();
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*prepared, output.get()));
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*transport, output.get()));
     PAIMON_RETURN_NOT_OK(RetainArrowArrayMemoryPool(output.get(), arrow_pool_));
     RecordBatchBuilder builder(output.get());
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RecordBatch> prepared_batch, builder.Finish());
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RecordBatch> transport_batch, builder.Finish());
     PAIMON_RETURN_NOT_OK(realtime_store_->Write(RealtimeWriteBatch{
-        std::move(prepared_batch), OffsetRange(next_offset_, next_offset_ + count)}));
+        std::move(transport_batch), OffsetRange(next_offset_, next_offset_ + count)}));
     next_offset_ += count;
     last_sequence_number_ += count;
     PAIMON_RETURN_NOT_OK(realtime_context_->AdvanceMaterializedMaxSequenceNumber(
@@ -239,16 +249,20 @@ Status RealtimePrimaryKeyWriter::FlushSegment(const std::shared_ptr<RealtimeSegm
                                               const OffsetRange& sealed_offsets) {
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
                            realtime_store_->CreateCommitReaders(segment));
-    PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> prepared_readers,
-                           PreparedKeyValueReaderFactory::CreateForCommit(
-                               std::move(readers), prepared_schema_, sealed_offsets, key_schema_,
-                               write_schema_, memory_pool_));
+    PAIMON_ASSIGN_OR_RAISE(
+        std::vector<std::unique_ptr<KeyValueRecordReader>> realtime_primary_key_readers,
+        RealtimePrimaryKeyReaderFactory::CreateForCommit(std::move(readers), transport_schema_,
+                                                         sealed_offsets, key_schema_, write_schema_,
+                                                         memory_pool_));
     std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
-    sorted_readers.reserve(prepared_readers.size());
-    for (std::unique_ptr<KeyValueRecordReader>& prepared_reader : prepared_readers) {
-        auto merge_function = std::make_unique<DeduplicateMergeFunction>(/*ignore_delete=*/false);
+    sorted_readers.reserve(realtime_primary_key_readers.size());
+    for (std::unique_ptr<KeyValueRecordReader>& realtime_primary_key_reader :
+         realtime_primary_key_readers) {
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<MergeFunction> merge_function,
+                               PrimaryKeyTableUtils::CreateMergeFunction(
+                                   write_schema_, trimmed_primary_keys_, options_, memory_pool_));
         sorted_readers.push_back(std::make_unique<MergedKeyValueRecordReader>(
-            std::move(prepared_reader), key_comparator_,
+            std::move(realtime_primary_key_reader), key_comparator_,
             std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge_function))));
     }
     return merge_tree_writer_->WriteSortedReadersToFiles(std::move(sorted_readers));
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.h b/src/paimon/core/realtime/realtime_primary_key_writer.h
index 4a26f930d..cdd3d889f 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.h
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.h
@@ -26,6 +26,7 @@
 #include <string>
 #include <vector>
 
+#include "paimon/core/core_options.h"
 #include "paimon/core/utils/batch_writer.h"
 #include "paimon/realtime/realtime_context.h"
 #include "paimon/realtime/realtime_store.h"
@@ -48,9 +49,9 @@ class RealtimePrimaryKeyWriter final : public BatchWriter {
     static Result<std::shared_ptr<RealtimePrimaryKeyWriter>> Create(
         const std::map<std::string, std::string>& partition, int32_t bucket,
         const std::shared_ptr<arrow::Schema>& write_schema,
-        const std::shared_ptr<arrow::Schema>& prepared_schema,
+        const std::shared_ptr<arrow::Schema>& transport_schema,
         const std::vector<std::string>& trimmed_primary_keys,
-        const std::shared_ptr<FieldsComparator>& key_comparator,
+        const std::shared_ptr<FieldsComparator>& key_comparator, const CoreOptions& options,
         const std::shared_ptr<RealtimeContextImpl>& realtime_context,
         const RealtimeStoreState& store_state, int64_t restored_max_sequence_number,
         const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
@@ -72,11 +73,12 @@ class RealtimePrimaryKeyWriter final : public BatchWriter {
                              const std::shared_ptr<RealtimeContextImpl>& realtime_context,
                              const RealtimePartitionBucket& partition_bucket,
                              const std::shared_ptr<arrow::Schema>& write_schema,
-                             const std::shared_ptr<arrow::Schema>& prepared_schema,
+                             const std::shared_ptr<arrow::Schema>& transport_schema,
                              const std::shared_ptr<arrow::Schema>& key_schema,
                              const std::vector<std::string>& trimmed_primary_keys,
                              const std::shared_ptr<FieldsComparator>& key_comparator,
-                             int64_t next_offset, int64_t last_sequence_number,
+                             const CoreOptions& options, int64_t next_offset,
+                             int64_t last_sequence_number,
                              const std::shared_ptr<MemoryPool>& memory_pool);
 
     Status FlushSegment(const std::shared_ptr<RealtimeSegmentHandle>& segment,
@@ -89,10 +91,11 @@ class RealtimePrimaryKeyWriter final : public BatchWriter {
     std::shared_ptr<RealtimeContextImpl> realtime_context_;
     RealtimePartitionBucket partition_bucket_;
     std::shared_ptr<arrow::Schema> write_schema_;
-    std::shared_ptr<arrow::Schema> prepared_schema_;
+    std::shared_ptr<arrow::Schema> transport_schema_;
     std::shared_ptr<arrow::Schema> key_schema_;
     std::vector<std::string> trimmed_primary_keys_;
     std::shared_ptr<FieldsComparator> key_comparator_;
+    CoreOptions options_;
     int64_t next_offset_;
     int64_t last_sequence_number_;
     std::mutex realtime_store_mutex_;
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index 110994331..afb852a6b 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -36,8 +36,8 @@
 #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
 #include "paimon/core/operation/merge_file_split_read.h"
 #include "paimon/core/operation/raw_file_split_read.h"
-#include "paimon/core/realtime/prepared_key_value_reader.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
+#include "paimon/core/realtime/realtime_primary_key_reader.h"
 #include "paimon/core/realtime/realtime_reader.h"
 #include "paimon/core/table/source/data_split_impl.h"
 #include "paimon/core/table/source/pk_count_reader.h"
@@ -56,55 +56,57 @@ struct ColumnarBatchContext;
 
 namespace {
 
-Result<std::shared_ptr<arrow::Schema>> CreatePreparedQuerySchema(
+Result<std::shared_ptr<arrow::Schema>> CreateRealtimePrimaryKeyQueryTransportSchema(
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema) {
-    arrow::FieldVector prepared_value_fields;
-    prepared_value_fields.reserve(key_schema->num_fields() + value_schema->num_fields());
+    arrow::FieldVector transport_value_fields;
+    transport_value_fields.reserve(key_schema->num_fields() + value_schema->num_fields());
     std::unordered_set<int32_t> field_ids;
     for (const std::shared_ptr<arrow::Field>& field : key_schema->fields()) {
         PAIMON_ASSIGN_OR_RAISE(int32_t field_id, NestedProjectionUtils::GetPaimonFieldId(field));
         if (field_ids.insert(field_id).second) {
-            prepared_value_fields.push_back(field);
+            transport_value_fields.push_back(field);
         }
     }
     for (const std::shared_ptr<arrow::Field>& field : value_schema->fields()) {
         PAIMON_ASSIGN_OR_RAISE(int32_t field_id, NestedProjectionUtils::GetPaimonFieldId(field));
         if (field_ids.insert(field_id).second) {
-            prepared_value_fields.push_back(field);
+            transport_value_fields.push_back(field);
         }
     }
-    return SpecialFields::PreparedKeyValueSchema(prepared_value_fields);
+    return RealtimePrimaryKeyLayout::CreateSchema(transport_value_fields);
 }
 
 Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateMemoryReaders(
     const std::shared_ptr<RealtimeSplit>& split, const RealtimePartitionBucketView& memory,
-    const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::shared_ptr<arrow::Schema>& transport_schema,
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
     const std::shared_ptr<FieldsComparator>& key_comparator,
     const std::shared_ptr<InternalReadContext>& context,
     const std::shared_ptr<MemoryPool>& memory_pool) {
     auto c_schema = std::make_unique<ArrowSchema>();
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*prepared_schema, c_schema.get()));
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*transport_schema, c_schema.get()));
     ScopeGuard schema_guard([schema = c_schema.get()]() { ArrowSchemaRelease(schema); });
     RealtimeQueryContext query_context{c_schema.get(), nullptr, false};
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> batch_readers,
                            memory.store->CreateQueryReaders(memory.read_view, 0, query_context));
-    PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> prepared_readers,
-                           PreparedKeyValueReaderFactory::CreateForQuery(
-                               std::move(batch_readers), prepared_schema,
-                               OffsetRange(split->CommittedEndOffset(), split->MemoryEndOffset()),
-                               key_schema, value_schema, memory_pool));
+    PAIMON_ASSIGN_OR_RAISE(
+        std::vector<std::unique_ptr<KeyValueRecordReader>> realtime_primary_key_readers,
+        RealtimePrimaryKeyReaderFactory::CreateForQuery(
+            std::move(batch_readers), transport_schema,
+            OffsetRange(split->CommittedEndOffset(), split->MemoryEndOffset()), key_schema,
+            value_schema, memory_pool));
     std::vector<std::unique_ptr<KeyValueRecordReader>> result;
-    result.reserve(prepared_readers.size());
-    for (std::unique_ptr<KeyValueRecordReader>& prepared_reader : prepared_readers) {
+    result.reserve(realtime_primary_key_readers.size());
+    for (std::unique_ptr<KeyValueRecordReader>& realtime_primary_key_reader :
+         realtime_primary_key_readers) {
         PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<MergeFunction> merge,
                                PrimaryKeyTableUtils::CreateMergeFunction(
                                    value_schema, context->GetTableSchema()->PrimaryKeys(),
                                    context->GetCoreOptions(), memory_pool));
         result.push_back(std::make_unique<MergedKeyValueRecordReader>(
-            std::move(prepared_reader), key_comparator,
+            std::move(realtime_primary_key_reader), key_comparator,
             std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge))));
     }
     return result;
@@ -112,17 +114,17 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateMemoryReaders(
 
 }  // namespace
 
-KeyValueTableRead::KeyValueTableRead(std::vector<std::unique_ptr<SplitRead>>&& split_reads,
-                                     const std::shared_ptr<FileStorePathFactory>& path_factory,
-                                     const std::shared_ptr<InternalReadContext>& context,
-                                     const std::shared_ptr<arrow::Schema>& prepared_query_schema,
-                                     const std::shared_ptr<MemoryPool>& memory_pool,
-                                     const std::shared_ptr<Executor>& executor)
+KeyValueTableRead::KeyValueTableRead(
+    std::vector<std::unique_ptr<SplitRead>>&& split_reads,
+    const std::shared_ptr<FileStorePathFactory>& path_factory,
+    const std::shared_ptr<InternalReadContext>& context,
+    const std::shared_ptr<arrow::Schema>& realtime_primary_key_transport_schema,
+    const std::shared_ptr<MemoryPool>& memory_pool, const std::shared_ptr<Executor>& executor)
     : TableRead(memory_pool),
       split_reads_(std::move(split_reads)),
       path_factory_(path_factory),
       context_(context),
-      prepared_query_schema_(prepared_query_schema),
+      realtime_primary_key_transport_schema_(realtime_primary_key_transport_schema),
       executor_(executor) {}
 
 Result<std::unique_ptr<TableRead>> KeyValueTableRead::Create(
@@ -136,17 +138,18 @@ Result<std::unique_ptr<TableRead>> KeyValueTableRead::Create(
     PAIMON_ASSIGN_OR_RAISE(
         std::unique_ptr<MergeFileSplitRead> merge_file_split_read,
         MergeFileSplitRead::Create(path_factory, context, memory_pool, executor));
-    std::shared_ptr<arrow::Schema> prepared_query_schema;
+    std::shared_ptr<arrow::Schema> realtime_primary_key_transport_schema;
     if (context->GetRealtimeContext()) {
-        PAIMON_ASSIGN_OR_RAISE(prepared_query_schema,
-                               CreatePreparedQuerySchema(merge_file_split_read->GetKeySchema(),
+        PAIMON_ASSIGN_OR_RAISE(
+            realtime_primary_key_transport_schema,
+            CreateRealtimePrimaryKeyQueryTransportSchema(merge_file_split_read->GetKeySchema(),
                                                          merge_file_split_read->GetValueSchema()));
     }
     split_reads.emplace_back(std::move(merge_file_split_read));
 
-    return std::unique_ptr<TableRead>(new KeyValueTableRead(std::move(split_reads), path_factory,
-                                                            context, prepared_query_schema,
-                                                            memory_pool, executor));
+    return std::unique_ptr<TableRead>(
+        new KeyValueTableRead(std::move(split_reads), path_factory, context,
+                              realtime_primary_key_transport_schema, memory_pool, executor));
 }
 
 void KeyValueTableRead::ForceKeepDelete(bool force_keep_delete) {
@@ -289,7 +292,7 @@ Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateRealtimeReader(
         if (merge_read) {
             PAIMON_ASSIGN_OR_RAISE(
                 std::vector<std::unique_ptr<KeyValueRecordReader>> memory_readers,
-                CreateMemoryReaders(realtime_split, memory, prepared_query_schema_,
+                CreateMemoryReaders(realtime_split, memory, realtime_primary_key_transport_schema_,
                                     merge_read->GetKeySchema(), merge_read->GetValueSchema(),
                                     merge_read->GetKeyComparator(), context_, GetMemoryPool()));
             PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> reader,
diff --git a/src/paimon/core/table/source/key_value_table_read.h b/src/paimon/core/table/source/key_value_table_read.h
index 54f802cf6..1dd59b016 100644
--- a/src/paimon/core/table/source/key_value_table_read.h
+++ b/src/paimon/core/table/source/key_value_table_read.h
@@ -59,7 +59,7 @@ class KeyValueTableRead : public TableRead {
     KeyValueTableRead(std::vector<std::unique_ptr<SplitRead>>&& split_reads,
                       const std::shared_ptr<FileStorePathFactory>& path_factory,
                       const std::shared_ptr<InternalReadContext>& context,
-                      const std::shared_ptr<arrow::Schema>& prepared_query_schema,
+                      const std::shared_ptr<arrow::Schema>& realtime_primary_key_transport_schema,
                       const std::shared_ptr<MemoryPool>& memory_pool,
                       const std::shared_ptr<Executor>& executor);
 
@@ -69,7 +69,7 @@ class KeyValueTableRead : public TableRead {
     std::vector<std::unique_ptr<SplitRead>> split_reads_;
     std::shared_ptr<FileStorePathFactory> path_factory_;
     std::shared_ptr<InternalReadContext> context_;
-    std::shared_ptr<arrow::Schema> prepared_query_schema_;
+    std::shared_ptr<arrow::Schema> realtime_primary_key_transport_schema_;
     std::shared_ptr<Executor> executor_;
     bool force_keep_delete_ = false;
 };

From d4ebe09a12e101172fec84a65c32bf90354ac3e7 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Fri, 28 Aug 2026 12:21:40 +0800
Subject: [PATCH 56/62] fix(realtime): resolve CI failures

---
 src/paimon/core/mergetree/merge_tree_writer.h |  3 +--
 .../realtime/arrow_realtime_store_test.cpp    |  8 --------
 .../primary_key_realtime_store_test.cpp       |  8 ++++----
 test/inte/realtime_write_inte_test.cpp        | 20 +++++++++++++------
 4 files changed, 19 insertions(+), 20 deletions(-)

diff --git a/src/paimon/core/mergetree/merge_tree_writer.h b/src/paimon/core/mergetree/merge_tree_writer.h
index 17a9dc51c..c2a9131e9 100644
--- a/src/paimon/core/mergetree/merge_tree_writer.h
+++ b/src/paimon/core/mergetree/merge_tree_writer.h
@@ -72,8 +72,7 @@ class MergeTreeWriter : public BatchWriter {
 
     /// Consumes readers whose complete streams are individually sorted by primary key and sequence
     /// number. Readers are closed on success or failure.
-    Status WriteSortedReadersToFiles(
-        std::vector<std::unique_ptr<KeyValueRecordReader>>&& readers);
+    Status WriteSortedReadersToFiles(std::vector<std::unique_ptr<KeyValueRecordReader>>&& readers);
 
     Status Compact(bool full_compaction) override;
 
diff --git a/src/paimon/core/realtime/arrow_realtime_store_test.cpp b/src/paimon/core/realtime/arrow_realtime_store_test.cpp
index d4bda2000..864b1f810 100644
--- a/src/paimon/core/realtime/arrow_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/arrow_realtime_store_test.cpp
@@ -271,14 +271,6 @@ TEST_F(ArrowRealtimeStoreTest, TestFullStatisticsPrunesNonMatchingBatch) {
     ASSERT_EQ(std::vector<int64_t>({0, 1}), ReadIds(unfiltered_batch));
 }
 
-TEST_F(ArrowRealtimeStoreTest, TestFactoryRejectsInvalidMode) {
-    ArrowRealtimeStoreFactory factory;
-    std::unique_ptr<ArrowSchema> write_schema = MakeReadSchema(schema_);
-    RealtimeStoreCreateRequest request{std::move(write_schema),
-                                       /*options=*/{}, pool_, static_cast<RealtimeStoreMode>(-1)};
-    ASSERT_NOK_WITH_MSG(factory.Create(std::move(request)), "invalid real-time store mode: -1");
-}
-
 TEST_F(ArrowRealtimeStoreTest, TestMissingStatisticsRetainsNonMatchingBatch) {
     ASSERT_OK(
         store_->Write(RealtimeWriteBatch{MakeBatch(R"([[0, "a"], [1, "b"]])"), OffsetRange(0, 2)}));
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index 301cc92a9..d3d5dff5e 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -361,11 +361,11 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryPoolOutlivesStoreReaderAndExport) {
     std::weak_ptr<TestingMemoryPool> pool_lifetime = pool;
     auto write_schema = std::make_unique<ArrowSchema>();
     ASSERT_TRUE(arrow::ExportSchema(*stored_schema, write_schema.get()).ok());
-    RealtimeStoreCreateRequest request{std::move(write_schema),
-                                       /*options=*/{}, pool, RealtimeStoreMode::PRIMARY_KEY};
     ArrowRealtimeStoreFactory factory;
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeStore> store, factory.Create(std::move(request)));
-    request.memory_pool.reset();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeStore> store,
+                         factory.Create(RealtimeStoreCreateRequest{
+                             std::move(write_schema),
+                             /*options=*/{}, pool, RealtimeStoreMode::PRIMARY_KEY}));
     ASSERT_OK(store->Write(
         RealtimeWriteBatch{MakeBatch(R"([[0, 1, 0, 7, "seven"]])"), OffsetRange(0, 1)}));
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index 73391b1a6..6b611300e 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -45,6 +45,7 @@
 #include "paimon/catalog/identifier.h"
 #include "paimon/commit_context.h"
 #include "paimon/common/factories/io_hook.h"
+#include "paimon/common/reader/reader_utils.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/path_util.h"
@@ -564,11 +565,18 @@ class CorruptingBatchReader final : public BatchReader {
     }
 
     void Close() override {
-        buffered_.reset();
+        ReleaseBuffered();
         delegate_->Close();
     }
 
  private:
+    void ReleaseBuffered() {
+        if (buffered_.has_value()) {
+            ReaderUtils::ReleaseReadBatch(std::move(buffered_.value()));
+            buffered_.reset();
+        }
+    }
+
     Result<ReadBatch> DropLast() {
         if (!buffered_.has_value()) {
             PAIMON_ASSIGN_OR_RAISE(ReadBatch first, delegate_->NextBatch());
@@ -579,7 +587,7 @@ class CorruptingBatchReader final : public BatchReader {
         }
         PAIMON_ASSIGN_OR_RAISE(ReadBatch next, delegate_->NextBatch());
         if (BatchReader::IsEofBatch(next)) {
-            buffered_.reset();
+            ReleaseBuffered();
             return MakeEofBatch();
         }
         ReadBatch result = std::move(buffered_.value());
@@ -2096,8 +2104,8 @@ TEST_F(RealtimeWriteInteTest, TestPkPartitionBucketRecovery) {
     }
     ASSERT_EQ(OffsetRange(0, 2), first_ranges.at(p0b0));
     ASSERT_EQ(OffsetRange(0, 3), first_ranges.at(p1b1));
-    ASSERT_EQ((std::make_pair<int64_t, int64_t>(0, 1)), first_sequences.at(p0b0));
-    ASSERT_EQ((std::make_pair<int64_t, int64_t>(0, 2)), first_sequences.at(p1b1));
+    ASSERT_EQ((std::pair<int64_t, int64_t>(0, 1)), first_sequences.at(p0b0));
+    ASSERT_EQ((std::pair<int64_t, int64_t>(0, 2)), first_sequences.at(p1b1));
     ASSERT_OK_AND_ASSIGN(int64_t first_snapshot_id,
                          Commit(first_progress, /*commit_identifier=*/0));
     ASSERT_OK(first_writer->RefreshCommittedSnapshot(first_snapshot_id));
@@ -2139,8 +2147,8 @@ TEST_F(RealtimeWriteInteTest, TestPkPartitionBucketRecovery) {
     }
     ASSERT_EQ(OffsetRange(2, 4), second_ranges.at(p0b0));
     ASSERT_EQ(OffsetRange(3, 5), second_ranges.at(p1b1));
-    ASSERT_EQ((std::make_pair<int64_t, int64_t>(2, 3)), second_sequences.at(p0b0));
-    ASSERT_EQ((std::make_pair<int64_t, int64_t>(3, 4)), second_sequences.at(p1b1));
+    ASSERT_EQ((std::pair<int64_t, int64_t>(2, 3)), second_sequences.at(p0b0));
+    ASSERT_EQ((std::pair<int64_t, int64_t>(3, 4)), second_sequences.at(p1b1));
     ASSERT_OK_AND_ASSIGN(int64_t second_snapshot_id,
                          Commit(second_progress, /*commit_identifier=*/1));
     ASSERT_OK(second_writer->RefreshCommittedSnapshot(second_snapshot_id));

From 49f0ea7bf3ab6a0f9b0d89839fcc8a898bc4011d Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Fri, 28 Aug 2026 14:02:37 +0800
Subject: [PATCH 57/62] test(realtime): remove redundant reader test code

---
 .../merged_key_value_record_reader_test.cpp   |  6 ----
 .../operation/merge_file_split_read_test.cpp  | 34 -------------------
 2 files changed, 40 deletions(-)

diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index d484b28ee..858e302e8 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -18,10 +18,7 @@
 
 #include "paimon/core/io/merged_key_value_record_reader.h"
 
-#include <cstdint>
 #include <memory>
-#include <optional>
-#include <string>
 #include <utility>
 #include <vector>
 
@@ -31,13 +28,10 @@
 #include "gtest/gtest.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
-#include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/fields_comparator.h"
 #include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
 #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
-#include "paimon/core/realtime/realtime_primary_key_reader.h"
 #include "paimon/memory/memory_pool.h"
-#include "paimon/realtime/offset_range.h"
 #include "paimon/testing/mock/mock_file_batch_reader.h"
 #include "paimon/testing/mock/mock_key_value_data_file_record_reader.h"
 #include "paimon/testing/utils/key_value_checker.h"
diff --git a/src/paimon/core/operation/merge_file_split_read_test.cpp b/src/paimon/core/operation/merge_file_split_read_test.cpp
index ad7c21b22..df9f5a02f 100644
--- a/src/paimon/core/operation/merge_file_split_read_test.cpp
+++ b/src/paimon/core/operation/merge_file_split_read_test.cpp
@@ -69,8 +69,6 @@ class FileSystem;
 namespace paimon::test {
 namespace {
 
-class TestingSplit : public Split {};
-
 class TrackingKeyValueRecordReader : public KeyValueRecordReader {
  public:
     explicit TrackingKeyValueRecordReader(int32_t* close_count) : close_count_(close_count) {}
@@ -764,38 +762,6 @@ TEST_P(MergeFileSplitReadTest, TestRealtimeReadConcatenatesOrderedDiskSections)
     batch_reader->Close();
 }
 
-TEST_F(MergeFileSplitReadTest, TestRealtimeReadValidatesDiskSplits) {
-    std::string path =
-        paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor";
-    ReadContextBuilder context_builder(path);
-    context_builder.SetReadFieldNames({"k0", "k1", "s1", "v0"});
-    context_builder.SetOptions(
-        {{Options::SEQUENCE_FIELD, "s0,s1"}, {Options::MERGE_ENGINE, "deduplicate"}});
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<ReadContext> read_context, context_builder.Finish());
-    std::shared_ptr<InternalReadContext> internal_context = CreateInternalReadContext(read_context);
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<MergeFileSplitRead> split_read,
-                         CreateMergeFileSplitRead(internal_context));
-
-    std::vector<std::shared_ptr<DataSplit>> prepared_splits = PrepareDataSplit();
-    std::shared_ptr<DataSplitImpl> first =
-        std::dynamic_pointer_cast<DataSplitImpl>(prepared_splits[0]);
-    ASSERT_NE(nullptr, first);
-
-    std::vector<std::shared_ptr<Split>> non_data_splits = {std::make_shared<TestingSplit>()};
-    ASSERT_NOK_WITH_MSG(split_read->CreateRealtimeReader(non_data_splits, {}),
-                        "merge input disk split is not a data split");
-
-    std::vector<std::shared_ptr<DataFileMeta>> deletion_data_files = first->DataFiles();
-    DataSplitImpl::Builder deletion_builder(first->Partition(), first->Bucket(),
-                                            first->BucketPath(), std::move(deletion_data_files));
-    ASSERT_OK_AND_ASSIGN(
-        std::shared_ptr<DataSplitImpl> deletion_split,
-        deletion_builder.WithDataDeletionFiles({std::nullopt}).RawConvertible(false).Build());
-    std::vector<std::shared_ptr<Split>> deletion_splits = {deletion_split};
-    ASSERT_NOK_WITH_MSG(split_read->CreateRealtimeReader(deletion_splits, {}),
-                        "deletion files must be empty or match data files");
-}
-
 TEST_F(MergeFileSplitReadTest, TestRealtimeReaderFailureClosesPluginReader) {
     std::string path =
         paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor";

From 813c6444e39dba4076018a34126a3e3b3dfc176c Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Fri, 28 Aug 2026 14:57:16 +0800
Subject: [PATCH 58/62] test(realtime): streamline realtime test coverage

---
 .../core/mergetree/merge_tree_writer_test.cpp |  47 -
 .../operation/merge_file_split_read_test.cpp  |  45 -
 .../primary_key_realtime_store_test.cpp       |  20 -
 .../core/realtime/realtime_context_test.cpp   |   6 +-
 .../realtime_primary_key_reader_test.cpp      |   1 -
 test/inte/realtime_write_inte_test.cpp        | 849 +-----------------
 6 files changed, 5 insertions(+), 963 deletions(-)

diff --git a/src/paimon/core/mergetree/merge_tree_writer_test.cpp b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
index 427c113e1..db2533fae 100644
--- a/src/paimon/core/mergetree/merge_tree_writer_test.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
@@ -345,29 +345,6 @@ TEST_P(MergeTreeWriterTest, TestSimple) {
     DataIncrement expected_data_increment({expected_data_file_meta}, /*deleted_files=*/{},
                                           /*changelog_files=*/{});
     ASSERT_EQ(expected_data_increment, commit_increment.GetNewFilesIncrement());
-
-    auto sorted_reader_array = std::dynamic_pointer_cast<arrow::StructArray>(
-        arrow::ipc::internal::json::ArrayFromJSON(write_type_, R"([
-      [2, 0, "Alice", 10, 0, 13.1],
-      [0, 0, "Lucy", 20, 1, 14.1],
-      [1, 0, "Paul", 20, 1, null]
-    ])")
-            .ValueOrDie());
-    auto sorted_reader_path_factory = std::make_shared<DataFilePathFactory>();
-    ASSERT_OK(sorted_reader_path_factory->Init(dir->Str() + "/sorted-readers", "orc",
-                                               options.DataFilePrefix(), nullptr));
-    ASSERT_OK_AND_ASSIGN(auto sorted_reader_writer,
-                         CreateMergeWriter(-1, dir->Str(), sorted_reader_path_factory, 1, options));
-    std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
-    sorted_readers.push_back(CreateSingleReader(sorted_reader_array));
-    ASSERT_OK(sorted_reader_writer->WriteSortedReadersToFiles(std::move(sorted_readers)));
-    ASSERT_OK_AND_ASSIGN(CommitIncrement sorted_reader_commit_increment,
-                         sorted_reader_writer->PrepareCommit(false));
-    ASSERT_OK(sorted_reader_writer->Close());
-    ASSERT_EQ(1, sorted_reader_commit_increment.GetNewFilesIncrement().NewFiles().size());
-    std::string sorted_reader_path = sorted_reader_path_factory->ToPath(
-        sorted_reader_commit_increment.GetNewFilesIncrement().NewFiles()[0]);
-    CheckFileContent(sorted_reader_path, expected_array);
 }
 
 TEST_P(MergeTreeWriterTest, TestWriteMultiBatch) {
@@ -449,30 +426,6 @@ TEST_P(MergeTreeWriterTest, TestWriteMultiBatch) {
     DataIncrement expected_data_increment({expected_data_file_meta}, /*deleted_files=*/{},
                                           /*changelog_files=*/{});
     ASSERT_EQ(expected_data_increment, commit_increment.GetNewFilesIncrement());
-
-    auto sorted_reader_array = std::dynamic_pointer_cast<arrow::StructArray>(
-        arrow::ipc::internal::json::ArrayFromJSON(write_type_, R"([
-      [16, 0, "Alice", 10, 0, 113.1],
-      [14, 0, "Lucy", 20, 1, 114.1],
-      [13, 0, "Paul", 20, 1, 15.1],
-      [15, 0, "Skye", 10, 0, 118.1]
-    ])")
-            .ValueOrDie());
-    auto sorted_reader_path_factory = std::make_shared<DataFilePathFactory>();
-    ASSERT_OK(sorted_reader_path_factory->Init(dir->Str() + "/sorted-readers", "orc",
-                                               options.DataFilePrefix(), nullptr));
-    ASSERT_OK_AND_ASSIGN(auto sorted_reader_writer,
-                         CreateMergeWriter(9, dir->Str(), sorted_reader_path_factory, 0, options));
-    std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
-    sorted_readers.push_back(CreateSingleReader(sorted_reader_array));
-    ASSERT_OK(sorted_reader_writer->WriteSortedReadersToFiles(std::move(sorted_readers)));
-    ASSERT_OK_AND_ASSIGN(CommitIncrement sorted_reader_commit_increment,
-                         sorted_reader_writer->PrepareCommit(false));
-    ASSERT_OK(sorted_reader_writer->Close());
-    ASSERT_EQ(1, sorted_reader_commit_increment.GetNewFilesIncrement().NewFiles().size());
-    std::string sorted_reader_path = sorted_reader_path_factory->ToPath(
-        sorted_reader_commit_increment.GetNewFilesIncrement().NewFiles()[0]);
-    CheckFileContent(sorted_reader_path, expected_array);
 }
 
 TEST_P(MergeTreeWriterTest, TestSortedReaders) {
diff --git a/src/paimon/core/operation/merge_file_split_read_test.cpp b/src/paimon/core/operation/merge_file_split_read_test.cpp
index df9f5a02f..d02120de4 100644
--- a/src/paimon/core/operation/merge_file_split_read_test.cpp
+++ b/src/paimon/core/operation/merge_file_split_read_test.cpp
@@ -52,7 +52,6 @@
 #include "paimon/executor.h"
 #include "paimon/fs/local/local_file_system.h"
 #include "paimon/memory/memory_pool.h"
-#include "paimon/metrics.h"
 #include "paimon/predicate/literal.h"
 #include "paimon/predicate/predicate_builder.h"
 #include "paimon/read_context.h"
@@ -67,29 +66,6 @@ class FileSystem;
 }  // namespace paimon
 
 namespace paimon::test {
-namespace {
-
-class TrackingKeyValueRecordReader : public KeyValueRecordReader {
- public:
-    explicit TrackingKeyValueRecordReader(int32_t* close_count) : close_count_(close_count) {}
-
-    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatch() override {
-        return std::unique_ptr<KeyValueRecordReader::Iterator>();
-    }
-
-    void Close() override {
-        ++(*close_count_);
-    }
-
-    std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return nullptr;
-    }
-
- private:
-    int32_t* close_count_;
-};
-
-}  // namespace
 
 // Parameter: min_heap/loser_tree; enable/disable IO prefetch; enable/disable multi thread row to
 // batch
@@ -762,27 +738,6 @@ TEST_P(MergeFileSplitReadTest, TestRealtimeReadConcatenatesOrderedDiskSections)
     batch_reader->Close();
 }
 
-TEST_F(MergeFileSplitReadTest, TestRealtimeReaderFailureClosesPluginReader) {
-    std::string path =
-        paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor";
-    ReadContextBuilder context_builder(path);
-    context_builder.SetReadFieldNames({"k0", "k1", "s1", "v0"});
-    context_builder.SetOptions(
-        {{Options::MERGE_ENGINE, "aggregation"}, {"fields.v0.aggregate-function", "unsupported"}});
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<ReadContext> read_context, context_builder.Finish());
-    std::shared_ptr<InternalReadContext> internal_context = CreateInternalReadContext(read_context);
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<MergeFileSplitRead> split_read,
-                         CreateMergeFileSplitRead(internal_context));
-
-    int32_t close_count = 0;
-    std::vector<std::unique_ptr<KeyValueRecordReader>> plugin_readers;
-    plugin_readers.push_back(std::make_unique<TrackingKeyValueRecordReader>(&close_count));
-
-    ASSERT_NOK_WITH_MSG(split_read->CreateRealtimeReader({}, std::move(plugin_readers)),
-                        "unsupported");
-    ASSERT_EQ(1, close_count);
-}
-
 TEST_P(MergeFileSplitReadTest, TestLookUp) {
     std::string path =
         paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor";
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index d3d5dff5e..7d6fa72f5 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -263,26 +263,6 @@ TEST(PrimaryKeyRealtimeStoreTest, TestSlicedReadersExportZeroOffsets) {
     AssertSlicedBatch(readers[0].get());
 }
 
-TEST(PrimaryKeyRealtimeStoreTest, TestCloseUnreadBatchReaders) {
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(TransportSchema(), GetDefaultPool()));
-    ASSERT_OK(
-        store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 10, 0, 1, "a"]])"), OffsetRange(0, 1)}));
-    ASSERT_OK(
-        store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 20, 1, 2, "b"]])"), OffsetRange(1, 2)}));
-    ASSERT_OK(
-        store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 30, 2, 3, "c"]])"), OffsetRange(2, 3)}));
-    ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
-                         store->SealForCommit());
-    ASSERT_TRUE(segment.has_value());
-    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
-                         store->CreateCommitReaders(segment.value()));
-    ASSERT_EQ(3, readers.size());
-    for (const std::unique_ptr<BatchReader>& reader : readers) {
-        reader->Close();
-    }
-}
-
 TEST(PrimaryKeyRealtimeStoreTest, TestReclaimKeepsReadView) {
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
                          PrimaryKeyRealtimeStore::Create(TransportSchema(), GetDefaultPool()));
diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp
index 82f831f6a..636cf12fc 100644
--- a/src/paimon/core/realtime/realtime_context_test.cpp
+++ b/src/paimon/core/realtime/realtime_context_test.cpp
@@ -217,8 +217,10 @@ TEST(RealtimeContextTest, TestReconcilesPrimaryKeyInitialSequence) {
     const std::map<std::string, std::string> partition = {{"dt", "2026-08-02"}};
     const RealtimePartitionBucket partition_bucket(partition, /*bucket=*/0);
 
-    ASSERT_OK(
-        GetOrCreateAppendStore(context, partition, 0, MakeWriteSchema(), {}, GetDefaultPool()));
+    ASSERT_OK(context->GetOrCreateRealtimeStore(
+        RealtimeStoreCreateRequest{
+            MakeWriteSchema(), /*options=*/{}, GetDefaultPool(), RealtimeStoreMode::PRIMARY_KEY},
+        partition_bucket));
 
     ASSERT_OK_AND_ASSIGN(int64_t first, context->AdvanceMaterializedMaxSequenceNumber(
                                             partition_bucket, /*max_sequence_number=*/4));
diff --git a/src/paimon/core/realtime/realtime_primary_key_reader_test.cpp b/src/paimon/core/realtime/realtime_primary_key_reader_test.cpp
index beb1a6435..89a39cd63 100644
--- a/src/paimon/core/realtime/realtime_primary_key_reader_test.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_reader_test.cpp
@@ -20,7 +20,6 @@
 
 #include <cstdint>
 #include <memory>
-#include <optional>
 #include <string>
 #include <utility>
 #include <vector>
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index 6b611300e..55227d841 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -29,7 +29,6 @@
 #include <map>
 #include <memory>
 #include <mutex>
-#include <new>
 #include <set>
 #include <string>
 #include <thread>
@@ -44,8 +43,6 @@
 #include "paimon/catalog/catalog.h"
 #include "paimon/catalog/identifier.h"
 #include "paimon/commit_context.h"
-#include "paimon/common/factories/io_hook.h"
-#include "paimon/common/reader/reader_utils.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/path_util.h"
@@ -84,88 +81,6 @@
 namespace paimon::test {
 namespace {
 
-bool HasSuffix(const std::string& value, const std::string& suffix) {
-    return value.size() >= suffix.size() &&
-           value.compare(value.size() - suffix.size(), suffix.size(), suffix) == 0;
-}
-
-Result<std::set<std::string>> ListPhysicalArtifacts(const std::shared_ptr<FileSystem>& file_system,
-                                                    const std::string& root) {
-    std::set<std::string> artifacts;
-    std::vector<std::string> directories = {root};
-    while (!directories.empty()) {
-        std::string directory = std::move(directories.back());
-        directories.pop_back();
-        std::vector<BasicFileStatus> statuses;
-        PAIMON_RETURN_NOT_OK(file_system->ListDir(directory, &statuses));
-        for (const BasicFileStatus& status : statuses) {
-            if (status.IsDir()) {
-                directories.push_back(status.GetPath());
-            } else if (HasSuffix(status.GetPath(), ".orc") ||
-                       HasSuffix(status.GetPath(), ".index") ||
-                       HasSuffix(status.GetPath(), ".channel")) {
-                artifacts.insert(status.GetPath());
-            }
-        }
-    }
-    return artifacts;
-}
-
-class FailAllocationMemoryPool final : public MemoryPool {
- public:
-    explicit FailAllocationMemoryPool(const std::shared_ptr<MemoryPool>& delegate)
-        : delegate_(delegate) {}
-
-    void FailAfterAllocations(int64_t successful_allocations) {
-        allocations_before_failure_.store(successful_allocations, std::memory_order_release);
-    }
-
-    void* Malloc(uint64_t size, uint64_t alignment = 0) override {
-        if (ShouldFail()) {
-            throw std::bad_alloc();
-        }
-        return delegate_->Malloc(size, alignment);
-    }
-
-    void* Realloc(void* p, size_t old_size, size_t new_size, uint64_t alignment = 0) override {
-        if (ShouldFail()) {
-            throw std::bad_alloc();
-        }
-        return delegate_->Realloc(p, old_size, new_size, alignment);
-    }
-
-    void Free(void* p, uint64_t size) override {
-        delegate_->Free(p, size);
-    }
-
-    void Free(void* p, uint64_t size, uint64_t alignment) override {
-        delegate_->Free(p, size, alignment);
-    }
-
-    uint64_t CurrentUsage() const override {
-        return delegate_->CurrentUsage();
-    }
-
-    uint64_t MaxMemoryUsage() const override {
-        return delegate_->MaxMemoryUsage();
-    }
-
- private:
-    bool ShouldFail() {
-        int64_t remaining = allocations_before_failure_.load(std::memory_order_acquire);
-        while (remaining >= 0) {
-            if (allocations_before_failure_.compare_exchange_weak(remaining, remaining - 1,
-                                                                  std::memory_order_acq_rel)) {
-                return remaining == 0;
-            }
-        }
-        return false;
-    }
-
-    std::shared_ptr<MemoryPool> delegate_;
-    std::atomic<int64_t> allocations_before_failure_{-1};
-};
-
 class TrackingRealtimeReadView final : public RealtimeReadView {
  public:
     explicit TrackingRealtimeReadView(std::shared_ptr<RealtimeReadView> delegate)
@@ -345,10 +260,6 @@ class CloseTrackingBatchReader final : public BatchReader {
 struct CloseTrackingReaderState {
     std::shared_ptr<std::atomic<int32_t>> query_close_count =
         std::make_shared<std::atomic<int32_t>>(0);
-    std::shared_ptr<std::atomic<int32_t>> commit_close_count =
-        std::make_shared<std::atomic<int32_t>>(0);
-    int32_t query_null_index = -1;
-    int32_t commit_null_index = -1;
 };
 
 class CloseTrackingRealtimeStore final : public DelegatingRealtimeStore {
@@ -357,18 +268,6 @@ class CloseTrackingRealtimeStore final : public DelegatingRealtimeStore {
                                const std::shared_ptr<CloseTrackingReaderState>& state)
         : DelegatingRealtimeStore(delegate), state_(state) {}
 
-    Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
-        const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
-        PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
-                               delegate_->CreateCommitReaders(segment));
-        for (std::unique_ptr<BatchReader>& reader : readers) {
-            reader = std::make_unique<CloseTrackingBatchReader>(std::move(reader),
-                                                                state_->commit_close_count);
-        }
-        PAIMON_RETURN_NOT_OK(InsertNullReader(state_->commit_null_index, &readers));
-        return readers;
-    }
-
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
         const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
         const RealtimeQueryContext& context) override {
@@ -378,302 +277,13 @@ class CloseTrackingRealtimeStore final : public DelegatingRealtimeStore {
             reader = std::make_unique<CloseTrackingBatchReader>(std::move(reader),
                                                                 state_->query_close_count);
         }
-        PAIMON_RETURN_NOT_OK(InsertNullReader(state_->query_null_index, &readers));
         return readers;
     }
 
  private:
-    static Status InsertNullReader(int32_t index,
-                                   std::vector<std::unique_ptr<BatchReader>>* readers) {
-        if (index < 0) {
-            return Status::OK();
-        }
-        if (index > static_cast<int32_t>(readers->size())) {
-            return Status::Invalid("null reader index exceeds reader count");
-        }
-        readers->insert(readers->begin() + index, nullptr);
-        return Status::OK();
-    }
-
     std::shared_ptr<CloseTrackingReaderState> state_;
 };
 
-class SplitBatchReader final : public BatchReader {
- public:
-    explicit SplitBatchReader(std::unique_ptr<BatchReader> delegate)
-        : delegate_(std::move(delegate)) {}
-
-    Result<ReadBatch> NextBatch() override {
-        while (!current_batch_ || next_row_ == current_batch_->length()) {
-            PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, delegate_->NextBatch());
-            if (BatchReader::IsEofBatch(batch)) {
-                return MakeEofBatch();
-            }
-            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-                std::shared_ptr<arrow::Array> array,
-                arrow::ImportArray(batch.first.get(), batch.second.get()));
-            if (!array || array->type_id() != arrow::Type::STRUCT) {
-                return Status::Invalid("split batch reader received a non-struct batch");
-            }
-            current_batch_ = std::dynamic_pointer_cast<arrow::StructArray>(array);
-            next_row_ = 0;
-        }
-        std::shared_ptr<arrow::Array> slice = current_batch_->Slice(next_row_, /*length=*/1);
-        ++next_row_;
-        auto output = std::make_unique<ArrowArray>();
-        auto schema = std::make_unique<ArrowSchema>();
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*slice, output.get(), schema.get()));
-        return ReadBatch(std::move(output), std::move(schema));
-    }
-
-    std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return delegate_->GetReaderMetrics();
-    }
-
-    void Close() override {
-        current_batch_.reset();
-        delegate_->Close();
-    }
-
- private:
-    std::unique_ptr<BatchReader> delegate_;
-    std::shared_ptr<arrow::StructArray> current_batch_;
-    int64_t next_row_ = 0;
-};
-
-class SplitCommitReaderRealtimeStore final : public DelegatingRealtimeStore {
- public:
-    explicit SplitCommitReaderRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate)
-        : DelegatingRealtimeStore(delegate) {}
-
-    Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
-        const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
-        PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
-                               delegate_->CreateCommitReaders(segment));
-        for (std::unique_ptr<BatchReader>& reader : readers) {
-            reader = std::make_unique<SplitBatchReader>(std::move(reader));
-        }
-        return readers;
-    }
-};
-
-class FailAfterPhysicalFileBatchReader final : public BatchReader {
- public:
-    FailAfterPhysicalFileBatchReader(std::unique_ptr<BatchReader> delegate,
-                                     const std::shared_ptr<FileSystem>& file_system,
-                                     std::string root, size_t baseline_artifact_count,
-                                     const std::shared_ptr<std::atomic<bool>>& saw_artifacts)
-        : delegate_(std::move(delegate)),
-          file_system_(file_system),
-          root_(std::move(root)),
-          baseline_artifact_count_(baseline_artifact_count),
-          saw_artifacts_(saw_artifacts) {}
-
-    Result<ReadBatch> NextBatch() override {
-        if (returned_batch_count_ < 4) {
-            ++returned_batch_count_;
-            return delegate_->NextBatch();
-        }
-        const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(30);
-        while (std::chrono::steady_clock::now() < deadline) {
-            PAIMON_ASSIGN_OR_RAISE(std::set<std::string> artifacts,
-                                   ListPhysicalArtifacts(file_system_, root_));
-            bool has_data = false;
-            for (const std::string& artifact : artifacts) {
-                has_data = has_data || HasSuffix(artifact, ".orc");
-            }
-            if (artifacts.size() > baseline_artifact_count_ && has_data) {
-                saw_artifacts_->store(true, std::memory_order_release);
-                return Status::IOError(
-                    "injected commit reader failure after physical file creation");
-            }
-            std::this_thread::sleep_for(std::chrono::milliseconds(1));
-        }
-        return Status::IOError("timed out waiting for partial physical files");
-    }
-
-    std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return delegate_->GetReaderMetrics();
-    }
-
-    void Close() override {
-        delegate_->Close();
-    }
-
- private:
-    std::unique_ptr<BatchReader> delegate_;
-    std::shared_ptr<FileSystem> file_system_;
-    std::string root_;
-    size_t baseline_artifact_count_;
-    std::shared_ptr<std::atomic<bool>> saw_artifacts_;
-    int32_t returned_batch_count_ = 0;
-};
-
-class FailAfterPhysicalFileRealtimeStore final : public DelegatingRealtimeStore {
- public:
-    FailAfterPhysicalFileRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
-                                       const std::shared_ptr<FileSystem>& file_system,
-                                       const std::string& root, size_t baseline_artifact_count,
-                                       const std::shared_ptr<std::atomic<bool>>& saw_artifacts)
-        : DelegatingRealtimeStore(delegate),
-          file_system_(file_system),
-          root_(root),
-          baseline_artifact_count_(baseline_artifact_count),
-          saw_artifacts_(saw_artifacts) {}
-
-    Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
-        const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
-        PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
-                               delegate_->CreateCommitReaders(segment));
-        if (readers.empty()) {
-            return Status::Invalid("commit reader failure test requires a reader");
-        }
-        readers[0] = std::make_unique<FailAfterPhysicalFileBatchReader>(
-            std::make_unique<SplitBatchReader>(std::move(readers[0])), file_system_, root_,
-            baseline_artifact_count_, saw_artifacts_);
-        return readers;
-    }
-
- private:
-    std::shared_ptr<FileSystem> file_system_;
-    std::string root_;
-    size_t baseline_artifact_count_;
-    std::shared_ptr<std::atomic<bool>> saw_artifacts_;
-};
-
-enum class ReaderMalformation { DROP_LAST, DUPLICATE_OFFSET, OUT_OF_RANGE_OFFSET };
-
-class CorruptingBatchReader final : public BatchReader {
- public:
-    CorruptingBatchReader(std::unique_ptr<BatchReader> delegate, ReaderMalformation malformation)
-        : delegate_(std::move(delegate)), malformation_(malformation) {}
-
-    Result<ReadBatch> NextBatch() override {
-        switch (malformation_) {
-            case ReaderMalformation::DROP_LAST:
-                return DropLast();
-            case ReaderMalformation::DUPLICATE_OFFSET:
-                return SubstituteOffset(/*offset=*/0);
-            case ReaderMalformation::OUT_OF_RANGE_OFFSET:
-                return SubstituteOffset(/*offset=*/-1);
-        }
-        return Status::Invalid("unknown commit reader malformation");
-    }
-
-    std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return delegate_->GetReaderMetrics();
-    }
-
-    void Close() override {
-        ReleaseBuffered();
-        delegate_->Close();
-    }
-
- private:
-    void ReleaseBuffered() {
-        if (buffered_.has_value()) {
-            ReaderUtils::ReleaseReadBatch(std::move(buffered_.value()));
-            buffered_.reset();
-        }
-    }
-
-    Result<ReadBatch> DropLast() {
-        if (!buffered_.has_value()) {
-            PAIMON_ASSIGN_OR_RAISE(ReadBatch first, delegate_->NextBatch());
-            if (BatchReader::IsEofBatch(first)) {
-                return MakeEofBatch();
-            }
-            buffered_ = std::move(first);
-        }
-        PAIMON_ASSIGN_OR_RAISE(ReadBatch next, delegate_->NextBatch());
-        if (BatchReader::IsEofBatch(next)) {
-            ReleaseBuffered();
-            return MakeEofBatch();
-        }
-        ReadBatch result = std::move(buffered_.value());
-        buffered_ = std::move(next);
-        return result;
-    }
-
-    Result<ReadBatch> SubstituteOffset(int64_t offset) {
-        PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, delegate_->NextBatch());
-        if (BatchReader::IsEofBatch(batch)) {
-            return batch;
-        }
-        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-            std::shared_ptr<arrow::Array> array,
-            arrow::ImportArray(batch.first.get(), batch.second.get()));
-        if (!array || array->type_id() != arrow::Type::STRUCT || array->length() == 0) {
-            return Status::Invalid("offset substitution requires a non-empty struct batch");
-        }
-        std::shared_ptr<arrow::StructArray> struct_array =
-            std::dynamic_pointer_cast<arrow::StructArray>(array);
-        std::shared_ptr<arrow::Int64Array> offsets =
-            std::dynamic_pointer_cast<arrow::Int64Array>(struct_array->field(2));
-        if (!offsets) {
-            return Status::Invalid("offset substitution requires an int64 REALTIME_OFFSET");
-        }
-        arrow::Int64Builder builder;
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Reserve(offsets->length()));
-        for (int64_t row = 0; row < offsets->length(); ++row) {
-            builder.UnsafeAppend(offset);
-        }
-        std::shared_ptr<arrow::Array> substituted_offsets;
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&substituted_offsets));
-        std::shared_ptr<arrow::ArrayData> substituted_data = struct_array->data()->Copy();
-        substituted_data->child_data[2] = substituted_offsets->data();
-        std::shared_ptr<arrow::Array> substituted = arrow::MakeArray(std::move(substituted_data));
-        auto output = std::make_unique<ArrowArray>();
-        auto schema = std::make_unique<ArrowSchema>();
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(
-            arrow::ExportArray(*substituted, output.get(), schema.get()));
-        return ReadBatch(std::move(output), std::move(schema));
-    }
-
-    std::unique_ptr<BatchReader> delegate_;
-    ReaderMalformation malformation_;
-    std::optional<ReadBatch> buffered_;
-};
-
-class MalformedCoverageRealtimeStore final : public DelegatingRealtimeStore {
- public:
-    MalformedCoverageRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
-                                   ReaderMalformation malformation)
-        : DelegatingRealtimeStore(delegate), malformation_(malformation) {}
-
-    Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
-        const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
-        PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
-                               delegate_->CreateCommitReaders(segment));
-        for (std::unique_ptr<BatchReader>& reader : readers) {
-            reader = std::make_unique<CorruptingBatchReader>(std::move(reader), malformation_);
-        }
-        return readers;
-    }
-
- private:
-    ReaderMalformation malformation_;
-};
-
-class MissingQueryOffsetRealtimeStore final : public DelegatingRealtimeStore {
- public:
-    explicit MissingQueryOffsetRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate)
-        : DelegatingRealtimeStore(delegate) {}
-
-    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
-        const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
-        const RealtimeQueryContext& context) override {
-        PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
-                               delegate_->CreateQueryReaders(view, offset_begin, context));
-        if (readers.empty()) {
-            return Status::Invalid("query offset drop requires a reader");
-        }
-        readers[0] = std::make_unique<CorruptingBatchReader>(std::move(readers[0]),
-                                                             ReaderMalformation::DROP_LAST);
-        return readers;
-    }
-};
-
 }  // namespace
 
 namespace {
@@ -1419,77 +1029,6 @@ class RealtimeWriteInteTest : public ::testing::Test {
         ASSERT_OK(writer->Close());
     }
 
-    void CheckVectorReaderRetry(bool primary_key) {
-        if (primary_key) {
-            CreatePkTable(/*partition_keys=*/{"pt"});
-        } else {
-            CreateTable(/*partition_keys=*/{"pt"});
-        }
-        auto close_state = std::make_shared<CloseTrackingReaderState>();
-        auto factory = MakeDecoratingFactory<CloseTrackingRealtimeStore>(close_state);
-        ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                             RealtimeContext::Create(factory));
-        ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                             CreateRealtimeWriter(realtime_context));
-        std::vector<Row> p0_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
-        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p0_batch,
-                             MakeBatch(p0_rows, /*partitioned=*/true));
-        ASSERT_OK(writer->Write(std::move(p0_batch)));
-        std::vector<Row> p1_rows = MakeRows(/*first_id=*/10, /*count=*/3, /*partition=*/"p1");
-        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p1_batch,
-                             MakeBatch(p1_rows, /*partitioned=*/true));
-        ASSERT_OK(writer->Write(std::move(p1_batch)));
-        ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
-                             CreatePlan(realtime_context, /*predicate=*/nullptr));
-        ASSERT_EQ(2, plan->Splits().size());
-
-        std::vector<std::shared_ptr<Split>> invalid_splits = plan->Splits();
-        std::shared_ptr<RealtimeSplit> second_split =
-            std::dynamic_pointer_cast<RealtimeSplit>(invalid_splits[1]);
-        ASSERT_NE(nullptr, second_split);
-        std::vector<std::shared_ptr<Split>> second_disk_splits = second_split->DiskSplits();
-        invalid_splits[1] = std::make_shared<RealtimeSplit>(
-            RealtimeSplit::kCurrentVersion + 1, second_split->SnapshotId(),
-            second_split->Partition(), second_split->Bucket(), std::move(second_disk_splits),
-            second_split->CommittedEndOffset(), second_split->MemoryEndOffset(),
-            second_split->OpaqueTicket());
-
-        ReadContextBuilder read_builder(table_path_);
-        read_builder.SetOptions(options_)
-            .SetReadFieldNames({"id", "payload", "pt"})
-            .WithRealtimeContext(realtime_context)
-            .WithMemoryPool(pool_);
-        ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
-        ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableRead> table_read,
-                             TableRead::Create(std::move(read_context)));
-        ASSERT_NOK_WITH_MSG(table_read->CreateReader(invalid_splits),
-                            "unsupported real-time split version");
-        ASSERT_EQ(1, close_state->query_close_count->load(std::memory_order_acquire));
-
-        std::vector<Row> expected_rows = p0_rows;
-        expected_rows.insert(expected_rows.end(), p1_rows.begin(), p1_rows.end());
-        ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(plan, realtime_context));
-        ASSERT_EQ(expected_rows, actual_rows);
-        ASSERT_OK(writer->Close());
-    }
-
-    void CheckPkRejectsReaderMalformation(ReaderMalformation malformation,
-                                          const std::string& expected_error) {
-        CreatePkTable();
-        auto factory = MakeDecoratingFactory<MalformedCoverageRealtimeStore>(malformation);
-        ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                             RealtimeContext::Create(factory));
-        ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                             CreateRealtimeWriter(realtime_context));
-        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
-                             MakeBatch({Row{1, "one", "p0"}, Row{2, "two", "p0"}},
-                                       /*partitioned=*/false));
-        ASSERT_OK(writer->Write(std::move(batch)));
-        ASSERT_NOK_WITH_MSG(writer->PrepareCommitWithProgress(/*commit_identifier=*/0),
-                            expected_error);
-        ASSERT_OK(writer->Close());
-    }
-
     std::unique_ptr<UniqueTestDirectory> dir_;
     std::string table_path_;
     std::string commit_user_ = "realtime_commit_user";
@@ -2197,26 +1736,6 @@ TEST_F(RealtimeWriteInteTest, TestPkRecovery) {
         RecordBatch::RowKind::INSERT, RecordBatch::RowKind::UPDATE_AFTER,
         RecordBatch::RowKind::DELETE, RecordBatch::RowKind::INSERT};
 
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> failed_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> failed_writer,
-                         CreateRealtimeWriter(failed_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> failed_batch,
-                         MakeBatch(mutations, /*partitioned=*/false, /*bucket=*/0, mutation_kinds));
-    ASSERT_OK(failed_writer->Write(std::move(failed_batch)));
-    IOHook* io_hook = IOHook::GetInstance();
-    ScopeGuard hook_guard([io_hook]() { io_hook->Clear(); });
-    io_hook->Reset(/*pos=*/0, IOHook::Mode::RETURN_ERROR);
-    Result<std::vector<RealtimeCommitProgress>> failed_prepare =
-        failed_writer->PrepareCommitWithProgress(/*commit_identifier=*/1);
-    io_hook->Clear();
-    ASSERT_TRUE(failed_prepare.status().IsIOError()) << failed_prepare.status().ToString();
-    ASSERT_OK(failed_writer->Close());
-    failed_writer.reset();
-    failed_context.reset();
-    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_failed_prepare, ReadRows());
-    ASSERT_EQ((std::vector<Row>{{99, "seed", "p0"}}), rows_after_failed_prepare);
-
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> first_context, RealtimeContext::Create());
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> first_writer,
                          CreateRealtimeWriter(first_context));
@@ -2379,9 +1898,8 @@ TEST_F(RealtimeWriteInteTest, TestPkCompaction) {
 
 TEST_F(RealtimeWriteInteTest, TestPkMultipleStoredBatchesMergeForQueryAndCommit) {
     CreatePkTable();
-    auto factory = MakeDecoratingFactory<SplitCommitReaderRealtimeStore>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create(factory));
+                         RealtimeContext::Create());
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                          CreateRealtimeWriter(realtime_context));
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
@@ -2410,37 +1928,6 @@ TEST_F(RealtimeWriteInteTest, TestPkMultipleStoredBatchesMergeForQueryAndCommit)
     ASSERT_OK(writer->Close());
 }
 
-TEST_F(RealtimeWriteInteTest, TestPkRejectsMalformedCoverage) {
-    CheckPkRejectsReaderMalformation(ReaderMalformation::DROP_LAST,
-                                     "commit readers did not cover the sealed range");
-}
-
-TEST_F(RealtimeWriteInteTest, TestPkRejectsDuplicateOffset) {
-    CheckPkRejectsReaderMalformation(ReaderMalformation::DUPLICATE_OFFSET,
-                                     "commit readers did not cover the sealed range");
-}
-
-TEST_F(RealtimeWriteInteTest, TestPkRejectsOutOfRangeOffset) {
-    CheckPkRejectsReaderMalformation(ReaderMalformation::OUT_OF_RANGE_OFFSET,
-                                     "offset is outside the sealed range");
-}
-
-TEST_F(RealtimeWriteInteTest, TestPkRejectsMissingQueryOffset) {
-    CreatePkTable();
-    auto factory = MakeDecoratingFactory<MissingQueryOffsetRealtimeStore>();
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create(factory));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                         CreateRealtimeWriter(realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
-                         MakeBatch({Row{1, "one", "p0"}, Row{2, "two", "p0"}},
-                                   /*partitioned=*/false));
-    ASSERT_OK(writer->Write(std::move(batch)));
-    ASSERT_NOK_WITH_MSG(ReadRows(realtime_context),
-                        "query readers did not cover the visible range");
-    ASSERT_OK(writer->Close());
-}
-
 TEST_F(RealtimeWriteInteTest, TestPkQueryReaderClose) {
     CreatePkTable();
     auto state = std::make_shared<CloseTrackingReaderState>();
@@ -2459,131 +1946,6 @@ TEST_F(RealtimeWriteInteTest, TestPkQueryReaderClose) {
     ASSERT_OK(writer->Close());
 }
 
-TEST_F(RealtimeWriteInteTest, TestPkQueryReaderCloseFailure) {
-    CreatePkTable();
-    auto state = std::make_shared<CloseTrackingReaderState>();
-    auto factory = MakeDecoratingFactory<CloseTrackingRealtimeStore>(state);
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create(factory));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                         CreateRealtimeWriter(realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
-                         MakeBatch({Row{1, "one", "p0"}}, /*partitioned=*/false));
-    ASSERT_OK(writer->Write(std::move(first_batch)));
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
-                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
-    ASSERT_EQ(1, progress.size());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
-                         MakeBatch({Row{2, "two", "p0"}}, /*partitioned=*/false));
-    ASSERT_OK(writer->Write(std::move(second_batch)));
-
-    for (int32_t null_index = 0; null_index <= 1; ++null_index) {
-        state->query_null_index = null_index;
-        ASSERT_NOK_WITH_MSG(CreateQueryReader(realtime_context),
-                            "PK real-time store returned a null query reader");
-        ASSERT_EQ((null_index + 1) * 2, state->query_close_count->load(std::memory_order_acquire));
-    }
-    ASSERT_OK(writer->Close());
-}
-
-TEST_F(RealtimeWriteInteTest, TestAppendQueryReaderCloseFailure) {
-    CreateTable(/*partition_keys=*/{});
-    auto state = std::make_shared<CloseTrackingReaderState>();
-    state->query_null_index = 1;
-    auto factory = MakeDecoratingFactory<CloseTrackingRealtimeStore>(state);
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create(factory));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                         CreateRealtimeWriter(realtime_context));
-    std::vector<Row> rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
-                         MakeBatch(rows, /*partitioned=*/false));
-    ASSERT_OK(writer->Write(std::move(batch)));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
-                         CreatePlan(realtime_context, /*predicate=*/nullptr));
-
-    ASSERT_NOK_WITH_MSG(CreateQueryReader(plan, realtime_context),
-                        "append-only real-time store returned a null query reader");
-    ASSERT_EQ(1, state->query_close_count->load(std::memory_order_acquire));
-
-    state->query_null_index = -1;
-    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(plan, realtime_context));
-    ASSERT_EQ(rows, actual_rows);
-    ASSERT_OK(writer->Close());
-}
-
-TEST_F(RealtimeWriteInteTest, TestPkCommitReaderCloseFailure) {
-    CreatePkTable();
-    auto state = std::make_shared<CloseTrackingReaderState>();
-    state->commit_null_index = 1;
-    auto factory = MakeDecoratingFactory<CloseTrackingRealtimeStore>(state);
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create(factory));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                         CreateRealtimeWriter(realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
-                         MakeBatch({Row{1, "one", "p0"}}, /*partitioned=*/false));
-    ASSERT_OK(writer->Write(std::move(batch)));
-
-    ASSERT_NOK_WITH_MSG(writer->PrepareCommitWithProgress(/*commit_identifier=*/0),
-                        "PK real-time store returned a null commit reader");
-    ASSERT_EQ(1, state->commit_close_count->load(std::memory_order_acquire));
-    ASSERT_OK(writer->Close());
-}
-
-TEST_F(RealtimeWriteInteTest, TestPkPrepareFailureCleansPartialPhysicalFiles) {
-    options_[Options::WRITE_BATCH_SIZE] = "1";
-    options_[Options::TARGET_FILE_ROW_NUM] = "1";
-    options_["file-index.bitmap.columns"] = "payload";
-    options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B";
-    CreatePkTable();
-    std::shared_ptr<FileSystem> file_system = dir_->GetFileSystem();
-    ASSERT_OK_AND_ASSIGN(std::set<std::string> baseline_artifacts,
-                         ListPhysicalArtifacts(file_system, dir_->Str()));
-    auto saw_artifacts = std::make_shared<std::atomic<bool>>(false);
-    auto factory = MakeDecoratingFactory<FailAfterPhysicalFileRealtimeStore>(
-        file_system, dir_->Str(), baseline_artifacts.size(), saw_artifacts);
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> failed_context,
-                         RealtimeContext::Create(factory));
-    WriteContextBuilder failed_builder(table_path_, commit_user_);
-    failed_builder.SetOptions(options_)
-        .WithStreamingMode(true)
-        .WithRealtimeContext(failed_context)
-        .WithTempDirectory(PathUtil::JoinPath(dir_->Str(), "tmp"));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> failed_write_context,
-                         failed_builder.Finish());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> failed_writer,
-                         FileStoreWrite::Create(std::move(failed_write_context)));
-
-    const std::vector<Row> wal = {
-        {1, "old", "p0"}, {1, "new", "p0"}, {2, "two", "p0"}, {2, "gone", "p0"}};
-    const std::vector<RecordBatch::RowKind> row_kinds = {
-        RecordBatch::RowKind::INSERT, RecordBatch::RowKind::UPDATE_AFTER,
-        RecordBatch::RowKind::INSERT, RecordBatch::RowKind::DELETE};
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> failed_batch,
-                         MakeBatch(wal, /*partitioned=*/false, /*bucket=*/0, row_kinds));
-    ASSERT_OK(failed_writer->Write(std::move(failed_batch)));
-    Result<std::vector<RealtimeCommitProgress>> failed_prepare =
-        failed_writer->PrepareCommitWithProgress(/*commit_identifier=*/0);
-    ASSERT_TRUE(failed_prepare.status().IsIOError()) << failed_prepare.status().ToString();
-    ASSERT_NE(std::string::npos,
-              failed_prepare.status().ToString().find(
-                  "injected commit reader failure after physical file creation"));
-    ASSERT_TRUE(saw_artifacts->load(std::memory_order_acquire));
-    ASSERT_OK_AND_ASSIGN(std::set<std::string> artifacts_after_abort,
-                         ListPhysicalArtifacts(file_system, dir_->Str()));
-    ASSERT_EQ(baseline_artifacts, artifacts_after_abort);
-
-    ASSERT_OK(failed_writer->Close());
-    failed_writer.reset();
-    failed_context.reset();
-
-    const std::vector<Row> expected_rows = {{1, "new", "p0"}};
-    ReplayPkWalAndCommit(wal, row_kinds, /*commit_identifier=*/0, expected_rows);
-    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap committed_offsets, ReadCommittedOffsets());
-    ASSERT_EQ(4, committed_offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
-}
-
 TEST_F(RealtimeWriteInteTest, TestRollingFilesPreserveProgress) {
     options_[Options::TARGET_FILE_ROW_NUM] = "10";
     CreateTable(/*partition_keys=*/{});
@@ -3015,44 +2377,6 @@ TEST_F(RealtimeWriteInteTest, TestPlanExcludesRowsWrittenAfterMemoryEndOffset) {
     ASSERT_OK(writer->Close());
 }
 
-TEST_F(RealtimeWriteInteTest, TestPkRejectsReversedVisibleOffsets) {
-    CreatePkTable();
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                         CreateRealtimeWriter(realtime_context));
-    std::vector<Row> rows = {{1, "one", "p0"}, {2, "two", "p0"}};
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
-                         MakeBatch(rows, /*partitioned=*/false));
-    ASSERT_OK(writer->Write(std::move(batch)));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
-                         CreatePlan(realtime_context, /*predicate=*/nullptr));
-    ASSERT_EQ(1, plan->Splits().size());
-    std::shared_ptr<RealtimeSplit> split =
-        std::dynamic_pointer_cast<RealtimeSplit>(plan->Splits()[0]);
-    ASSERT_NE(nullptr, split);
-    std::vector<std::shared_ptr<Split>> disk_splits = split->DiskSplits();
-    std::vector<std::shared_ptr<Split>> invalid_splits = {std::make_shared<RealtimeSplit>(
-        split->Version(), split->SnapshotId(), split->Partition(), split->Bucket(),
-        std::move(disk_splits), split->MemoryEndOffset() + 1, split->MemoryEndOffset(),
-        split->OpaqueTicket())};
-
-    ReadContextBuilder read_builder(table_path_);
-    read_builder.SetOptions(options_)
-        .SetReadFieldNames({"id", "payload", "pt"})
-        .WithRealtimeContext(realtime_context)
-        .WithMemoryPool(pool_);
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableRead> table_read,
-                         TableRead::Create(std::move(read_context)));
-    ASSERT_NOK_WITH_MSG(table_read->CreateReader(invalid_splits),
-                        "memory end offset precedes committed end offset");
-
-    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(plan, realtime_context));
-    ASSERT_EQ(rows, actual_rows);
-    ASSERT_OK(writer->Close());
-}
-
 TEST_F(RealtimeWriteInteTest, TestReadFailsAfterRealtimeSplitTicketExpires) {
     options_[Options::REALTIME_READ_VIEW_TTL] = "10 ms";
     CreateTable(/*partition_keys=*/{});
@@ -3134,14 +2458,6 @@ TEST_F(RealtimeWriteInteTest, TestFailedReaderCreationPreservesRealtimeSplitTick
     ASSERT_OK(writer->Close());
 }
 
-TEST_F(RealtimeWriteInteTest, TestAppendVectorRetry) {
-    CheckVectorReaderRetry(/*primary_key=*/false);
-}
-
-TEST_F(RealtimeWriteInteTest, TestPkVectorRetry) {
-    CheckVectorReaderRetry(/*primary_key=*/true);
-}
-
 TEST_F(RealtimeWriteInteTest, TestCloseWriterKeepsContextReadable) {
     CreateTable(/*partition_keys=*/{});
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
@@ -4630,167 +3946,4 @@ TEST_F(RealtimeWriteInteTest, TestRestoreOffsetFromCommittedSnapshot) {
     ASSERT_EQ(5, second_committed_offsets.at(partition_bucket));
 }
 
-TEST_F(RealtimeWriteInteTest, TestPkWriteFailureRecovery) {
-    CreatePkTable();
-    const std::vector<Row> seed_rows = {{99, "seed", "p0"}};
-    ReplayPkWalAndCommit(seed_rows, /*row_kinds=*/{}, /*commit_identifier=*/0, seed_rows);
-
-    const std::vector<Row> wal = {
-        {1, "one", "p0"}, {1, "one-new", "p0"}, {2, "deleted", "p0"}, {3, "three", "p0"}};
-    const std::vector<RecordBatch::RowKind> row_kinds = {
-        RecordBatch::RowKind::INSERT, RecordBatch::RowKind::UPDATE_AFTER,
-        RecordBatch::RowKind::DELETE, RecordBatch::RowKind::INSERT};
-    std::shared_ptr<FailAllocationMemoryPool> failing_pool =
-        std::make_shared<FailAllocationMemoryPool>(pool_);
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> failed_context,
-                         RealtimeContext::Create());
-    WriteContextBuilder failed_builder(table_path_, commit_user_);
-    failed_builder.SetOptions(options_)
-        .WithStreamingMode(true)
-        .WithRealtimeContext(failed_context)
-        .WithMemoryPool(failing_pool);
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> failed_write_context,
-                         failed_builder.Finish());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> failed_writer,
-                         FileStoreWrite::Create(std::move(failed_write_context)));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> empty_batch,
-                         MakeUnpartitionedBatchFromJson("[]"));
-    ASSERT_OK(failed_writer->Write(std::move(empty_batch)));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> failed_batch,
-                         MakeBatch(wal, /*partitioned=*/false, /*bucket=*/0, row_kinds));
-    failing_pool->FailAfterAllocations(1);
-    Status failed_write = failed_writer->Write(std::move(failed_batch));
-    ASSERT_TRUE(failed_write.IsOutOfMemory()) << failed_write.ToString();
-    ASSERT_OK(failed_writer->Close());
-    failed_writer.reset();
-    failed_context.reset();
-
-    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_failure, ReadRows());
-    ASSERT_EQ(seed_rows, rows_after_failure);
-
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> replay_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> replay_writer,
-                         CreateRealtimeWriter(replay_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> replay_batch,
-                         MakeBatch(wal, /*partitioned=*/false, /*bucket=*/0, row_kinds));
-    ASSERT_OK(replay_writer->Write(std::move(replay_batch)));
-    const std::vector<Row> expected_rows = {
-        {1, "one-new", "p0"}, {3, "three", "p0"}, {99, "seed", "p0"}};
-    ASSERT_OK_AND_ASSIGN(std::vector<Row> replayed_rows, ReadRows(replay_context));
-    ASSERT_EQ(expected_rows, replayed_rows);
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
-                         replay_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
-    ASSERT_EQ(1, progress.size());
-    ASSERT_EQ(OffsetRange(1, 5), progress[0].offset_range);
-    ASSERT_EQ(1, NewFiles(progress).size());
-    ASSERT_EQ(1, NewFiles(progress)[0]->delete_row_count);
-    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, /*commit_identifier=*/1));
-    ASSERT_OK(replay_writer->RefreshCommittedSnapshot(snapshot_id));
-    ASSERT_OK(replay_writer->Close());
-    replay_writer.reset();
-    replay_context.reset();
-
-    ASSERT_OK_AND_ASSIGN(std::vector<Row> persisted_rows, ReadRows());
-    ASSERT_EQ(expected_rows, persisted_rows);
-    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap committed_offsets, ReadCommittedOffsets());
-    ASSERT_EQ(5, committed_offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
-}
-
-TEST_F(RealtimeWriteInteTest, TestPkExternalCommitRecovery) {
-    CreatePkTable();
-    const std::vector<Row> seed_rows = {{99, "seed", "p0"}};
-    ReplayPkWalAndCommit(seed_rows, /*row_kinds=*/{}, /*commit_identifier=*/0, seed_rows);
-
-    const std::vector<Row> wal = {
-        {1, "one", "p0"}, {1, "one-new", "p0"}, {2, "deleted", "p0"}, {3, "three", "p0"}};
-    const std::vector<RecordBatch::RowKind> row_kinds = {
-        RecordBatch::RowKind::INSERT, RecordBatch::RowKind::UPDATE_AFTER,
-        RecordBatch::RowKind::DELETE, RecordBatch::RowKind::INSERT};
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> failed_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> failed_writer,
-                         CreateRealtimeWriter(failed_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> failed_batch,
-                         MakeBatch(wal, /*partitioned=*/false, /*bucket=*/0, row_kinds));
-    ASSERT_OK(failed_writer->Write(std::move(failed_batch)));
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> failed_progress,
-                         failed_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
-    ASSERT_EQ(1, failed_progress.size());
-    CommitContextBuilder commit_builder(table_path_, commit_user_);
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
-                         commit_builder.SetOptions(options_).Finish());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> commit,
-                         FileStoreCommit::Create(std::move(commit_context)));
-    IOHook* io_hook = IOHook::GetInstance();
-    ScopeGuard hook_guard([io_hook]() { io_hook->Clear(); });
-    io_hook->Reset(/*pos=*/0, IOHook::Mode::RETURN_ERROR);
-    Result<int64_t> failed_commit =
-        commit->CommitWithProgress(failed_progress, /*commit_identifier=*/1,
-                                   /*watermark=*/std::nullopt);
-    io_hook->Clear();
-    ASSERT_TRUE(failed_commit.status().IsIOError()) << failed_commit.status().ToString();
-    ASSERT_OK(failed_writer->Close());
-    failed_writer.reset();
-    failed_context.reset();
-    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_failure, ReadRows());
-    ASSERT_EQ(seed_rows, rows_after_failure);
-
-    const std::vector<Row> expected_rows = {
-        {1, "one-new", "p0"}, {3, "three", "p0"}, {99, "seed", "p0"}};
-    ReplayPkWalAndCommit(wal, row_kinds, /*commit_identifier=*/1, expected_rows);
-}
-
-TEST_F(RealtimeWriteInteTest, TestPkRefreshRecovery) {
-    CreatePkTable();
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> failed_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> failed_writer,
-                         CreateRealtimeWriter(failed_context));
-
-    const std::vector<Row> base_rows = {{1, "one", "p0"}, {2, "two", "p0"}};
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> base_batch,
-                         MakeBatch(base_rows, /*partitioned=*/false));
-    ASSERT_OK(failed_writer->Write(std::move(base_batch)));
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> base_progress,
-                         failed_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
-    ASSERT_OK_AND_ASSIGN(int64_t base_snapshot_id, Commit(base_progress, /*commit_identifier=*/0));
-    ASSERT_OK(failed_writer->RefreshCommittedSnapshot(base_snapshot_id));
-
-    const std::vector<Row> committed_wal = {
-        {1, "one-new", "p0"}, {2, "deleted", "p0"}, {3, "three", "p0"}};
-    const std::vector<RecordBatch::RowKind> committed_kinds = {RecordBatch::RowKind::UPDATE_AFTER,
-                                                               RecordBatch::RowKind::DELETE,
-                                                               RecordBatch::RowKind::INSERT};
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<RecordBatch> committed_batch,
-        MakeBatch(committed_wal, /*partitioned=*/false, /*bucket=*/0, committed_kinds));
-    ASSERT_OK(failed_writer->Write(std::move(committed_batch)));
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> committed_progress,
-                         failed_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
-    ASSERT_OK_AND_ASSIGN(int64_t committed_snapshot_id,
-                         Commit(committed_progress, /*commit_identifier=*/1));
-
-    const std::vector<Row> replay_wal = {{4, "four", "p0"}};
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> replay_batch,
-                         MakeBatch(replay_wal, /*partitioned=*/false));
-    ASSERT_OK(failed_writer->Write(std::move(replay_batch)));
-    IOHook* io_hook = IOHook::GetInstance();
-    ScopeGuard hook_guard([io_hook]() { io_hook->Clear(); });
-    io_hook->Reset(/*pos=*/0, IOHook::Mode::RETURN_ERROR);
-    Status failed_refresh = failed_writer->RefreshCommittedSnapshot(committed_snapshot_id);
-    io_hook->Clear();
-    ASSERT_TRUE(failed_refresh.IsIOError()) << failed_refresh.ToString();
-    ASSERT_OK(failed_writer->Close());
-    failed_writer.reset();
-    failed_context.reset();
-    const std::vector<Row> committed_rows = {{1, "one-new", "p0"}, {3, "three", "p0"}};
-    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_failure, ReadRows());
-    ASSERT_EQ(committed_rows, rows_after_failure);
-
-    const std::vector<Row> expected_rows = {
-        {1, "one-new", "p0"}, {3, "three", "p0"}, {4, "four", "p0"}};
-    ReplayPkWalAndCommit(replay_wal, /*row_kinds=*/{}, /*commit_identifier=*/2, expected_rows);
-}
-
 }  // namespace paimon::test

From e3574185c75c8b01a985c9e50917fdd0026f5401 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Fri, 28 Aug 2026 16:09:29 +0800
Subject: [PATCH 59/62] fix(style): apply clang-format

---
 src/paimon/core/realtime/realtime_context_test.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp
index 636cf12fc..c9531eda5 100644
--- a/src/paimon/core/realtime/realtime_context_test.cpp
+++ b/src/paimon/core/realtime/realtime_context_test.cpp
@@ -218,8 +218,8 @@ TEST(RealtimeContextTest, TestReconcilesPrimaryKeyInitialSequence) {
     const RealtimePartitionBucket partition_bucket(partition, /*bucket=*/0);
 
     ASSERT_OK(context->GetOrCreateRealtimeStore(
-        RealtimeStoreCreateRequest{
-            MakeWriteSchema(), /*options=*/{}, GetDefaultPool(), RealtimeStoreMode::PRIMARY_KEY},
+        RealtimeStoreCreateRequest{MakeWriteSchema(), /*options=*/{}, GetDefaultPool(),
+                                   RealtimeStoreMode::PRIMARY_KEY},
         partition_bucket));
 
     ASSERT_OK_AND_ASSIGN(int64_t first, context->AdvanceMaterializedMaxSequenceNumber(

From 020a14dc9dfad35b4bc07034684ffe599a8382a7 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Fri, 28 Aug 2026 18:01:18 +0800
Subject: [PATCH 60/62] docs: preserve comments across refactoring

---
 include/paimon/realtime/arrow_realtime_store_factory.h | 1 +
 include/paimon/realtime/realtime_store.h               | 1 +
 src/paimon/core/mergetree/merge_tree_writer.cpp        | 7 ++++++-
 src/paimon/core/operation/merge_file_split_read.cpp    | 2 ++
 4 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/include/paimon/realtime/arrow_realtime_store_factory.h b/include/paimon/realtime/arrow_realtime_store_factory.h
index 153d524d4..b3fa630de 100644
--- a/include/paimon/realtime/arrow_realtime_store_factory.h
+++ b/include/paimon/realtime/arrow_realtime_store_factory.h
@@ -26,6 +26,7 @@ namespace paimon {
 /// Factory for Paimon's default Arrow-backed `RealtimeStore`.
 class PAIMON_EXPORT ArrowRealtimeStoreFactory : public RealtimeStoreFactory {
  public:
+    /// Creates an Arrow-backed store for one partition and bucket.
     Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override;
 };
 
diff --git a/include/paimon/realtime/realtime_store.h b/include/paimon/realtime/realtime_store.h
index 81e64b9fd..6ae81f1f4 100644
--- a/include/paimon/realtime/realtime_store.h
+++ b/include/paimon/realtime/realtime_store.h
@@ -189,6 +189,7 @@ class PAIMON_EXPORT RealtimeStoreFactory {
  public:
     virtual ~RealtimeStoreFactory() = default;
 
+    /// Creates a store configured with the supplied schema, statistics, options, and memory pool.
     /// Creates a store for the requested table mode.
     /// The factory consumes `request`, including ownership of `request.write_schema`.
     virtual Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) = 0;
diff --git a/src/paimon/core/mergetree/merge_tree_writer.cpp b/src/paimon/core/mergetree/merge_tree_writer.cpp
index 75623fd96..c9eb44bba 100644
--- a/src/paimon/core/mergetree/merge_tree_writer.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer.cpp
@@ -172,16 +172,20 @@ Status MergeTreeWriter::WriteSortedReadersToFiles(
         }
     }
 
+    // 2. prepare loser tree sort merge reader
     auto sort_merge_reader = std::make_unique<SortMergeReaderWithLoserTree>(
         std::move(readers), key_comparator_, user_defined_seq_comparator_, merge_function_wrapper_);
     raw_readers_guard.Release();
+    // 3. project key value to arrow array
     auto create_consumer = [target_schema = write_schema_, pool = pool_]()
         -> Result<std::unique_ptr<RowToArrowArrayConverter<KeyValue, KeyValueBatch>>> {
         return KeyValueMetaProjectionConsumer::Create(target_schema, pool);
     };
+    // consumer batch size is WriteBatchSize
     auto async_key_value_producer_consumer =
         std::make_unique<AsyncKeyValueProducerAndConsumer<KeyValue, KeyValueBatch>>(
-            std::move(sort_merge_reader), create_consumer, options_.GetWriteBatchSize(), 1, pool_);
+            std::move(sort_merge_reader), create_consumer, options_.GetWriteBatchSize(),
+            /*projection_thread_num=*/1, pool_);
     ScopeGuard async_readers_guard([&]() -> void { async_key_value_producer_consumer->Close(); });
     std::unique_ptr<RollingFileWriter<KeyValueBatch, std::shared_ptr<DataFileMeta>>> rolling_writer;
     PAIMON_ASSIGN_OR_RAISE(rolling_writer, CreateRollingRowWriter());
@@ -309,6 +313,7 @@ Status MergeTreeWriter::FlushWriteBuffer(bool wait_for_latest_compaction,
             wait_for_latest_compaction = true;
         }
         auto cleanup_guard = ScopeGuard([&]() { write_buffer_->Clear(); });
+        // 1. flush write buffer to get sorted readers
         PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
                                write_buffer_->CreateReaders());
         PAIMON_RETURN_NOT_OK(WriteSortedReadersToFiles(std::move(readers)));
diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp
index 3bcc705aa..1f96c195c 100644
--- a/src/paimon/core/operation/merge_file_split_read.cpp
+++ b/src/paimon/core/operation/merge_file_split_read.cpp
@@ -663,6 +663,7 @@ MergeFileSplitRead::CreateRecordReadersForSection(
     std::vector<std::unique_ptr<KeyValueRecordReader>> record_readers;
     record_readers.reserve(section.size());
     for (const SortedRun& run : section) {
+        // no overlap in a run
         PAIMON_ASSIGN_OR_RAISE(
             std::unique_ptr<KeyValueRecordReader> run_reader,
             CreateReaderForRun(partition, run, dv_factory, predicate, data_file_path_factory));
@@ -722,6 +723,7 @@ Result<std::unique_ptr<SortMergeReader>> MergeFileSplitRead::CreateSortMergeRead
     DeletionVector::Factory dv_factory, const std::shared_ptr<Predicate>& predicate,
     const std::shared_ptr<DataFilePathFactory>& data_file_path_factory, bool drop_delete,
     const std::shared_ptr<MergeFunctionWrapper<KeyValue>>& merge_function_wrapper) {
+    // with overlap in one section
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> record_readers,
                            CreateRecordReadersForSection(section, partition, dv_factory, predicate,
                                                          data_file_path_factory));

From a77b5f4608f60f0731311680cb2f4c095c035e56 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Fri, 28 Aug 2026 18:47:30 +0800
Subject: [PATCH 61/62] test(realtime): streamline primary key reader coverage

---
 .../key_value_file_store_write_test.cpp       |  18 ++-
 .../primary_key_realtime_store_test.cpp       |  85 +++----------
 .../realtime_primary_key_reader_test.cpp      |  23 ++++
 test/inte/realtime_write_inte_test.cpp        | 116 +-----------------
 4 files changed, 54 insertions(+), 188 deletions(-)

diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp
index 01ba8459d..88b848eea 100644
--- a/src/paimon/core/operation/key_value_file_store_write_test.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp
@@ -50,6 +50,7 @@
 #include "paimon/core/io/data_file_meta.h"
 #include "paimon/core/operation/restore_files.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
+#include "paimon/core/realtime/realtime_primary_key_reader.h"
 #include "paimon/core/stats/simple_stats.h"
 #include "paimon/core/table/sink/commit_message_impl.h"
 #include "paimon/file_store_commit.h"
@@ -257,17 +258,12 @@ class KeyValueFileStoreWriteTest : public ::testing::Test {
         if (views.size() != 1) {
             return Status::Invalid("expected exactly one real-time store");
         }
-        std::shared_ptr<arrow::Schema> transport_schema = arrow::schema({
-            DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())
-                ->WithNullable(false),
-            DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
-                ->WithNullable(false),
-            DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()),
-            DataField::ConvertDataFieldToArrowField(
-                DataField(0, arrow::field("id", arrow::int64(), false))),
-            DataField::ConvertDataFieldToArrowField(
-                DataField(1, arrow::field("value", arrow::utf8()))),
-        });
+        arrow::FieldVector value_fields = {DataField::ConvertDataFieldToArrowField(DataField(
+                                               0, arrow::field("id", arrow::int64(), false))),
+                                           DataField::ConvertDataFieldToArrowField(
+                                               DataField(1, arrow::field("value", arrow::utf8())))};
+        std::shared_ptr<arrow::Schema> transport_schema =
+            RealtimePrimaryKeyLayout::CreateSchema(value_fields);
         auto c_schema = std::make_unique<ArrowSchema>();
         PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*transport_schema, c_schema.get()));
         RealtimeQueryContext query_context{c_schema.get(), nullptr, false};
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index 7d6fa72f5..ed80db275 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -18,7 +18,6 @@
 
 #include "paimon/core/realtime/primary_key_realtime_store.h"
 
-#include <cstddef>
 #include <cstdint>
 #include <memory>
 #include <optional>
@@ -29,10 +28,10 @@
 #include "arrow/api.h"
 #include "arrow/c/bridge.h"
 #include "arrow/ipc/json_simple.h"
-#include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
+#include "paimon/core/realtime/realtime_primary_key_reader.h"
 #include "paimon/macros.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/realtime/arrow_realtime_store_factory.h"
@@ -50,28 +49,17 @@ std::shared_ptr<arrow::Field> FieldWithId(const std::string& name,
 }
 
 std::shared_ptr<arrow::Schema> TransportSchema() {
-    return arrow::schema(
-        {DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
-         DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
-             ->WithNullable(false),
-         DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()),
-         DataField::ConvertDataFieldToArrowField(DataField(0, arrow::field("id", arrow::int64()))),
-         DataField::ConvertDataFieldToArrowField(
-             DataField(1, arrow::field("value", arrow::utf8())))});
+    return RealtimePrimaryKeyLayout::CreateSchema(
+        {FieldWithId("id", arrow::int64(), 0), FieldWithId("value", arrow::utf8(), 1)});
 }
 
 std::shared_ptr<arrow::Schema> NestedTransportSchema() {
-    return arrow::schema(
-        {DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
-         DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
-             ->WithNullable(false),
-         DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()),
-         DataField::ConvertDataFieldToArrowField(DataField(0, arrow::field("id", arrow::int64()))),
-         DataField::ConvertDataFieldToArrowField(DataField(
-             1,
-             arrow::field("value",
-                          arrow::struct_({arrow::field("name", arrow::utf8()),
-                                          arrow::field("items", arrow::list(arrow::int32()))}))))});
+    return RealtimePrimaryKeyLayout::CreateSchema(
+        {FieldWithId("id", arrow::int64(), 0),
+         FieldWithId("value",
+                     arrow::struct_({arrow::field("name", arrow::utf8()),
+                                     arrow::field("items", arrow::list(arrow::int32()))}),
+                     1)});
 }
 
 std::unique_ptr<RecordBatch> MakeBatch(const std::string& json) {
@@ -125,36 +113,6 @@ Result<std::string> ReadJson(const std::vector<std::unique_ptr<BatchReader>>& re
     return result->ToString();
 }
 
-class TestingMemoryPool final : public MemoryPool {
- public:
-    void* Malloc(uint64_t size, uint64_t alignment) override {
-        return delegate_->Malloc(size, alignment);
-    }
-
-    void* Realloc(void* pointer, size_t old_size, size_t new_size, uint64_t alignment) override {
-        return delegate_->Realloc(pointer, old_size, new_size, alignment);
-    }
-
-    void Free(void* pointer, uint64_t size) override {
-        delegate_->Free(pointer, size);
-    }
-
-    void Free(void* pointer, uint64_t size, uint64_t alignment) override {
-        delegate_->Free(pointer, size, alignment);
-    }
-
-    uint64_t CurrentUsage() const override {
-        return delegate_->CurrentUsage();
-    }
-
-    uint64_t MaxMemoryUsage() const override {
-        return delegate_->MaxMemoryUsage();
-    }
-
- private:
-    std::unique_ptr<MemoryPool> delegate_ = GetMemoryPool();
-};
-
 TEST(PrimaryKeyRealtimeStoreTest, TestWriteAndSealValidation) {
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
                          PrimaryKeyRealtimeStore::Create(TransportSchema(), GetDefaultPool()));
@@ -337,8 +295,8 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderPerStoredBatch) {
 
 TEST(PrimaryKeyRealtimeStoreTest, TestQueryPoolOutlivesStoreReaderAndExport) {
     const std::shared_ptr<arrow::Schema> stored_schema = TransportSchema();
-    std::shared_ptr<TestingMemoryPool> pool = std::make_shared<TestingMemoryPool>();
-    std::weak_ptr<TestingMemoryPool> pool_lifetime = pool;
+    std::shared_ptr<MemoryPool> pool = GetMemoryPool();
+    std::weak_ptr<MemoryPool> pool_lifetime = pool;
     auto write_schema = std::make_unique<ArrowSchema>();
     ASSERT_TRUE(arrow::ExportSchema(*stored_schema, write_schema.get()).ok());
     ArrowRealtimeStoreFactory factory;
@@ -381,16 +339,13 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderProjectsNestedFields) {
     const std::shared_ptr<arrow::Field> stored_b = FieldWithId("b", arrow::int32(), 11);
     const std::shared_ptr<arrow::Field> stored_x = FieldWithId("x", arrow::int32(), 20);
     const std::shared_ptr<arrow::Field> stored_y = FieldWithId("y", arrow::int32(), 21);
-    arrow::FieldVector stored_fields = {
-        DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
-        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
-            ->WithNullable(false),
-        DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()),
+    arrow::FieldVector stored_value_fields = {
         FieldWithId("id", arrow::int64(), 0),
         FieldWithId("profile", arrow::struct_({stored_profile_a}), 1),
         FieldWithId("items", arrow::list(arrow::struct_({stored_a, stored_b})), 2),
         FieldWithId("attrs", arrow::map(arrow::utf8(), arrow::struct_({stored_x, stored_y})), 3)};
-    std::shared_ptr<arrow::Schema> stored_schema = arrow::schema(std::move(stored_fields));
+    std::shared_ptr<arrow::Schema> stored_schema =
+        RealtimePrimaryKeyLayout::CreateSchema(stored_value_fields);
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
                          PrimaryKeyRealtimeStore::Create(stored_schema, GetDefaultPool()));
     ASSERT_OK(store->Write(RealtimeWriteBatch{
@@ -401,14 +356,14 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderProjectsNestedFields) {
         OffsetRange(0, 1)}));
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
 
-    arrow::FieldVector requested_fields(stored_schema->fields().begin(),
-                                        stored_schema->fields().begin() + 3);
-    requested_fields.push_back(FieldWithId("profile", arrow::struct_({stored_profile_a}), 1));
-    requested_fields.push_back(
+    arrow::FieldVector requested_value_fields;
+    requested_value_fields.push_back(FieldWithId("profile", arrow::struct_({stored_profile_a}), 1));
+    requested_value_fields.push_back(
         FieldWithId("items", arrow::list(arrow::struct_({stored_b, stored_a})), 2));
-    requested_fields.push_back(
+    requested_value_fields.push_back(
         FieldWithId("attrs", arrow::map(arrow::utf8(), arrow::struct_({stored_y, stored_x})), 3));
-    std::shared_ptr<arrow::Schema> requested_schema = arrow::schema(std::move(requested_fields));
+    std::shared_ptr<arrow::Schema> requested_schema =
+        RealtimePrimaryKeyLayout::CreateSchema(requested_value_fields);
     auto c_schema = std::make_unique<ArrowSchema>();
     ASSERT_TRUE(arrow::ExportSchema(*requested_schema, c_schema.get()).ok());
     RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
diff --git a/src/paimon/core/realtime/realtime_primary_key_reader_test.cpp b/src/paimon/core/realtime/realtime_primary_key_reader_test.cpp
index 89a39cd63..bd925cd96 100644
--- a/src/paimon/core/realtime/realtime_primary_key_reader_test.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_reader_test.cpp
@@ -685,4 +685,27 @@ TEST_F(RealtimePrimaryKeyReaderTest, TestFactoryFailureClosesReaders) {
     ASSERT_EQ(factory_failure_close_count, 1);
 }
 
+TEST_F(RealtimePrimaryKeyReaderTest, TestQueryReaderClose) {
+    std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32())),
+                                           DataField(1, arrow::field("v0", arrow::int32()))};
+    std::shared_ptr<arrow::Schema> value_schema =
+        DataField::ConvertDataFieldsToArrowSchema(value_fields);
+    std::shared_ptr<arrow::Schema> key_schema = arrow::schema({value_schema->field(0)});
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema(value_schema->fields());
+    std::shared_ptr<arrow::DataType> transport_type = arrow::struct_(transport_schema->fields());
+    std::shared_ptr<arrow::Array> transport_array =
+        arrow::ipc::internal::json::ArrayFromJSON(transport_type, R"([[0, 10, 0, 1, 100]])")
+            .ValueOrDie();
+
+    int32_t close_count = 0;
+    auto batch_reader = std::make_unique<TrackingBatchReader>(
+        std::make_unique<MockFileBatchReader>(transport_array, transport_type, 1), &close_count);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
+                         CreateRealtimePrimaryKeyQueryReaderForTest(
+                             std::move(batch_reader), transport_schema, OffsetRange(0, 1),
+                             key_schema, value_schema, pool_));
+    reader->Close();
+    ASSERT_EQ(close_count, 1);
+}
+
 }  // namespace paimon::test
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index 55227d841..4e28f286f 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -50,6 +50,7 @@
 #include "paimon/core/core_options.h"
 #include "paimon/core/operation/commit/realtime_commit_properties.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
+#include "paimon/core/realtime/realtime_primary_key_reader.h"
 #include "paimon/core/schema/schema_manager.h"
 #include "paimon/core/schema/table_schema.h"
 #include "paimon/core/table/sink/commit_message_impl.h"
@@ -98,32 +99,6 @@ class TrackingRealtimeReadView final : public RealtimeReadView {
     std::shared_ptr<RealtimeReadView> delegate_;
 };
 
-class ReadViewCheckingBatchReader final : public BatchReader {
- public:
-    ReadViewCheckingBatchReader(std::unique_ptr<BatchReader> delegate,
-                                std::weak_ptr<RealtimeReadView> read_view)
-        : delegate_(std::move(delegate)), read_view_(std::move(read_view)) {}
-
-    Result<ReadBatch> NextBatch() override {
-        if (read_view_.expired()) {
-            return Status::Invalid("real-time read view was released before reader completion");
-        }
-        return delegate_->NextBatch();
-    }
-
-    std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return delegate_->GetReaderMetrics();
-    }
-
-    void Close() override {
-        delegate_->Close();
-    }
-
- private:
-    std::unique_ptr<BatchReader> delegate_;
-    std::weak_ptr<RealtimeReadView> read_view_;
-};
-
 class DelegatingRealtimeStore : public RealtimeStore {
  public:
     explicit DelegatingRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate)
@@ -219,13 +194,7 @@ class QueryTrackingRealtimeStore final : public DelegatingRealtimeStore {
         if (!tracking_view) {
             return Status::Invalid("query tracking store received an unexpected read view");
         }
-        PAIMON_ASSIGN_OR_RAISE(
-            std::vector<std::unique_ptr<BatchReader>> readers,
-            delegate_->CreateQueryReaders(tracking_view->Delegate(), offset_begin, context));
-        for (std::unique_ptr<BatchReader>& reader : readers) {
-            reader = std::make_unique<ReadViewCheckingBatchReader>(std::move(reader), view);
-        }
-        return readers;
+        return delegate_->CreateQueryReaders(tracking_view->Delegate(), offset_begin, context);
     }
 
  private:
@@ -233,57 +202,6 @@ class QueryTrackingRealtimeStore final : public DelegatingRealtimeStore {
     std::shared_ptr<std::weak_ptr<RealtimeReadView>> query_view_;
 };
 
-class CloseTrackingBatchReader final : public BatchReader {
- public:
-    CloseTrackingBatchReader(std::unique_ptr<BatchReader> delegate,
-                             const std::shared_ptr<std::atomic<int32_t>>& close_count)
-        : delegate_(std::move(delegate)), close_count_(close_count) {}
-
-    Result<ReadBatch> NextBatch() override {
-        return delegate_->NextBatch();
-    }
-
-    std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return delegate_->GetReaderMetrics();
-    }
-
-    void Close() override {
-        close_count_->fetch_add(1, std::memory_order_release);
-        delegate_->Close();
-    }
-
- private:
-    std::unique_ptr<BatchReader> delegate_;
-    std::shared_ptr<std::atomic<int32_t>> close_count_;
-};
-
-struct CloseTrackingReaderState {
-    std::shared_ptr<std::atomic<int32_t>> query_close_count =
-        std::make_shared<std::atomic<int32_t>>(0);
-};
-
-class CloseTrackingRealtimeStore final : public DelegatingRealtimeStore {
- public:
-    CloseTrackingRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
-                               const std::shared_ptr<CloseTrackingReaderState>& state)
-        : DelegatingRealtimeStore(delegate), state_(state) {}
-
-    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
-        const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
-        const RealtimeQueryContext& context) override {
-        PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
-                               delegate_->CreateQueryReaders(view, offset_begin, context));
-        for (std::unique_ptr<BatchReader>& reader : readers) {
-            reader = std::make_unique<CloseTrackingBatchReader>(std::move(reader),
-                                                                state_->query_close_count);
-        }
-        return readers;
-    }
-
- private:
-    std::shared_ptr<CloseTrackingReaderState> state_;
-};
-
 }  // namespace
 
 namespace {
@@ -846,18 +764,10 @@ class RealtimeWriteInteTest : public ::testing::Test {
             return Status::Invalid("expected a table schema");
         }
         auto read_schema = std::make_unique<ArrowSchema>();
-        arrow::FieldVector requested_fields = {
-            DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())
-                ->WithNullable(false),
-            DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
-                ->WithNullable(false),
-            DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset())};
         std::shared_ptr<arrow::Schema> value_schema =
             DataField::ConvertDataFieldsToArrowSchema(table_schema.value()->Fields());
-        requested_fields.insert(requested_fields.end(), value_schema->fields().begin(),
-                                value_schema->fields().end());
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(
-            arrow::ExportSchema(*arrow::schema(requested_fields), read_schema.get()));
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(
+            *RealtimePrimaryKeyLayout::CreateSchema(value_schema->fields()), read_schema.get()));
         ScopeGuard schema_guard([schema = read_schema.get()]() { ArrowSchemaRelease(schema); });
         RealtimeQueryContext query_context{read_schema.get(), /*predicate=*/nullptr,
                                            /*enable_predicate_pushdown=*/false};
@@ -1928,24 +1838,6 @@ TEST_F(RealtimeWriteInteTest, TestPkMultipleStoredBatchesMergeForQueryAndCommit)
     ASSERT_OK(writer->Close());
 }
 
-TEST_F(RealtimeWriteInteTest, TestPkQueryReaderClose) {
-    CreatePkTable();
-    auto state = std::make_shared<CloseTrackingReaderState>();
-    auto factory = MakeDecoratingFactory<CloseTrackingRealtimeStore>(state);
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create(factory));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                         CreateRealtimeWriter(realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
-                         MakeBatch({Row{1, "one", "p0"}}, /*partitioned=*/false));
-    ASSERT_OK(writer->Write(std::move(batch)));
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> reader, CreateQueryReader(realtime_context));
-    reader->Close();
-    ASSERT_EQ(1, state->query_close_count->load(std::memory_order_acquire));
-    ASSERT_OK(writer->Close());
-}
-
 TEST_F(RealtimeWriteInteTest, TestRollingFilesPreserveProgress) {
     options_[Options::TARGET_FILE_ROW_NUM] = "10";
     CreateTable(/*partition_keys=*/{});

From a51b207c72e260e1e145b0194517afce069333bb Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Fri, 28 Aug 2026 19:43:09 +0800
Subject: [PATCH 62/62] test(realtime): remove redundant close coverage

---
 .../core/mergetree/merge_tree_writer.cpp      |  4 ++--
 .../realtime_primary_key_reader_test.cpp      | 23 -------------------
 2 files changed, 2 insertions(+), 25 deletions(-)

diff --git a/src/paimon/core/mergetree/merge_tree_writer.cpp b/src/paimon/core/mergetree/merge_tree_writer.cpp
index c9eb44bba..dde9aaeba 100644
--- a/src/paimon/core/mergetree/merge_tree_writer.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer.cpp
@@ -172,11 +172,11 @@ Status MergeTreeWriter::WriteSortedReadersToFiles(
         }
     }
 
-    // 2. prepare loser tree sort merge reader
+    // prepare loser tree sort merge reader
     auto sort_merge_reader = std::make_unique<SortMergeReaderWithLoserTree>(
         std::move(readers), key_comparator_, user_defined_seq_comparator_, merge_function_wrapper_);
     raw_readers_guard.Release();
-    // 3. project key value to arrow array
+    // project key value to arrow array
     auto create_consumer = [target_schema = write_schema_, pool = pool_]()
         -> Result<std::unique_ptr<RowToArrowArrayConverter<KeyValue, KeyValueBatch>>> {
         return KeyValueMetaProjectionConsumer::Create(target_schema, pool);
diff --git a/src/paimon/core/realtime/realtime_primary_key_reader_test.cpp b/src/paimon/core/realtime/realtime_primary_key_reader_test.cpp
index bd925cd96..89a39cd63 100644
--- a/src/paimon/core/realtime/realtime_primary_key_reader_test.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_reader_test.cpp
@@ -685,27 +685,4 @@ TEST_F(RealtimePrimaryKeyReaderTest, TestFactoryFailureClosesReaders) {
     ASSERT_EQ(factory_failure_close_count, 1);
 }
 
-TEST_F(RealtimePrimaryKeyReaderTest, TestQueryReaderClose) {
-    std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32())),
-                                           DataField(1, arrow::field("v0", arrow::int32()))};
-    std::shared_ptr<arrow::Schema> value_schema =
-        DataField::ConvertDataFieldsToArrowSchema(value_fields);
-    std::shared_ptr<arrow::Schema> key_schema = arrow::schema({value_schema->field(0)});
-    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema(value_schema->fields());
-    std::shared_ptr<arrow::DataType> transport_type = arrow::struct_(transport_schema->fields());
-    std::shared_ptr<arrow::Array> transport_array =
-        arrow::ipc::internal::json::ArrayFromJSON(transport_type, R"([[0, 10, 0, 1, 100]])")
-            .ValueOrDie();
-
-    int32_t close_count = 0;
-    auto batch_reader = std::make_unique<TrackingBatchReader>(
-        std::make_unique<MockFileBatchReader>(transport_array, transport_type, 1), &close_count);
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
-                         CreateRealtimePrimaryKeyQueryReaderForTest(
-                             std::move(batch_reader), transport_schema, OffsetRange(0, 1),
-                             key_schema, value_schema, pool_));
-    reader->Close();
-    ASSERT_EQ(close_count, 1);
-}
-
 }  // namespace paimon::test