diff --git a/docs/source/deployment/ssd/ssd-offload.md b/docs/source/deployment/ssd/ssd-offload.md index 34d2dd28a9..cd10b3c618 100644 --- a/docs/source/deployment/ssd/ssd-offload.md +++ b/docs/source/deployment/ssd/ssd-offload.md @@ -185,6 +185,32 @@ Groups multiple objects into bucket files. Reduces filesystem overhead, supports Best for: general-purpose use, large-scale deployments. +#### Explicit-Delete GC (tombstone compaction) + +To enable SSD space reclamation via `Remove`/`BatchRemove` (without LRU +eviction deleting live keys), set: + +| Environment Variable | Required Value | Description | +|---|---|---| +| `MOONCAKE_OFFLOAD_BUCKET_EVICTION_POLICY` | `lru` | Keeps `last_access_ns_` updated for GC cold-bucket selection | +| `MOONCAKE_OFFLOAD_DISABLE_SSD_EVICTION` | `true` | Makes `PrepareEviction` a no-op so no bucket is ever evicted by LRU | + +GC tuning (optional): + +| Environment Variable | Default | Description | +|---|---|---| +| `MOONCAKE_OFFLOAD_BUCKET_GC_ENABLE` | `true` | Enable background tombstone compaction | +| `MOONCAKE_OFFLOAD_BUCKET_GC_INTERVAL_MS` | `1000` | GC scan interval | +| `MOONCAKE_OFFLOAD_BUCKET_GC_DELETED_RATIO` | `0.25` | Compact bucket when deleted bytes / data size >= this | +| `MOONCAKE_OFFLOAD_BUCKET_GC_HIGH_WATERMARK_RATIO` | `0.90` | Force compaction of any tombstone bucket when total size / max >= this | +| `MOONCAKE_OFFLOAD_BUCKET_GC_MAX_BUCKETS_PER_ROUND` | `1` | Max old buckets collected per GC round for cross-bucket merge | +| `MOONCAKE_OFFLOAD_BUCKET_GC_MERGE_ENABLE` | `true` | Enable cross-bucket merge: collect live keys from multiple tombstone buckets into one new bucket. When false, each bucket is compacted independently | + +**Important:** Only keys removed via `Remove`/`BatchRemove` are reclaimed. +`RemoveByRegex`/`RemoveAll` do not trigger this GC. When no tombstone space +is reclaimable and SSD is full, `BatchOffload` returns an error instead of +deleting live keys. + ### `file_per_key_storage_backend` Stores each object in an individual file. Simple and easy to inspect, but generates many small files at scale. diff --git a/docs/yh/deployment.md b/docs/yh/deployment.md new file mode 100644 index 0000000000..df0a0879a2 --- /dev/null +++ b/docs/yh/deployment.md @@ -0,0 +1,777 @@ + + + + +# POC项目Mooncake部署文档 + +## 1 ETCD环境搭建 +### 1.1 安装 + +```bash +wget --no-check-certificate https://github.com/etcd-io/etcd/releases/download/v3.5.9/etcd-v3.5.9-linux-arm64.tar.gz +tar -xzvf etcd-v3.5.9-linux-arm64.tar.gz +rm etcd-v3.5.9-linux-arm64.tar.gz +mv etcd-v3.5.9-linux-arm64/ etcd +cd etcd/ +cp etcd etcdctl /usr/local/bin/ +``` + +### 1.2 配置 + +#### 1.2.1 方法1，一同创建（推荐） + +##### 1.2.1.1 设置system脚本 + +```bash +vim /etc/systemd/system/etcd.service +``` + +`ETCD_NAME, ETCD_INITIAL_ADVERTISE_PEER_URLS, ETCD_ADVERTISE_CLIENT_URLS`根据本机实际配置 +`ETCD_INITIAL_CLUSTER`配置集群全部节点的地址 + +``` +[Unit] +Description=etcd key-value store +After=network.target + +[Service] +Type=notify +Environment=ETCD_NAME= +Environment=ETCD_DATA_DIR=/var/lib/etcd +Environment=ETCD_LISTEN_PEER_URLS=http://0.0.0.0:2380 +Environment=ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379 +Environment=ETCD_INITIAL_ADVERTISE_PEER_URLS=http://:2380 +Environment=ETCD_ADVERTISE_CLIENT_URLS=http://:2379 +Environment=ETCD_INITIAL_CLUSTER="=http://:2380,=http://:2380" +Environment=ETCD_INITIAL_CLUSTER_STATE="new" +Environment=ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster" +ExecStart=/usr/local/bin/etcd +Restart=on-failure +RestartSec=5 +User=root + +[Install] +WantedBy=multi-user.target +``` + +##### 1.2.1.2 清除缓存，应用配置，启动etcd服务 + +为所有节点配置完成后，启动全部服务 + +```bash +systemctl disable etcd +systemctl stop etcd +rm -rf /var/lib/etcd +systemctl daemon-reload +systemctl start etcd +systemctl enable etcd +``` + +##### 1.2.1.3 查看集群 + +```bash +etcdctl --endpoints=http://:2379 member list --write-out=table +``` + +#### 1.2.2 方法2，逐个创建 + +##### 1.2.2.1 设置system脚本 + +```bash +vim /etc/systemd/system/etcd.service +``` + +主机 （ip换为本机ip） + +``` +[Unit] +Description=etcd key-value store +After=network.target + +[Service] +Type=notify +Environment=ETCD_NAME= +Environment=ETCD_DATA_DIR=/var/lib/etcd +Environment=ETCD_LISTEN_PEER_URLS=http://0.0.0.0:2380 +Environment=ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379 +Environment=ETCD_INITIAL_ADVERTISE_PEER_URLS=http://:2380 +Environment=ETCD_ADVERTISE_CLIENT_URLS=http://:2379 +Environment=ETCD_INITIAL_CLUSTER="=http://:2380" +Environment=ETCD_INITIAL_CLUSTER_STATE="new" +Environment=ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster" +ExecStart=/usr/local/bin/etcd +Restart=on-failure +RestartSec=5 +User=root + +[Install] +WantedBy=multi-user.target +``` + +##### 1.2.2.2 清除缓存，应用配置，启动etcd服务 + +```bash +systemctl stop etcd +rm -rf /var/lib/etcd +systemctl daemon-reload +systemctl start etcd +``` + +##### 1.2.2.3 添加节点 + +```bash +etcdctl --endpoints=http://:2379 member add --peer-urls=http://:2380 +``` + +有如下打印则为添加成功 + +``` +[root@slot1 system]# etcdctl --endpoints=http://:2379 member add --peer-urls=http://:2380 +Member 6a5201d77502faa1 added to cluster 592c5fccdb3ab88c + +ETCD_NAME="" +ETCD_INITIAL_CLUSTER="=http://:2380,=http://:2380" +ETCD_INITIAL_ADVERTISE_PEER_URLS="http://:2380" +ETCD_INITIAL_CLUSTER_STATE="existing" +``` + +##### 1.2.2.4 将打印的部分内容复制到新节点etcd的配置文件 + +例：的/etc/systemd/system/etcd.service配置文件 + +``` +[Unit] +Description=etcd key-value store +After=network.target + +[Service] +Type=notify +Environment=ETCD_NAME= +Environment=ETCD_DATA_DIR=/var/lib/etcd +Environment=ETCD_LISTEN_PEER_URLS=http://0.0.0.0:2380 +Environment=ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379 +Environment=ETCD_INITIAL_ADVERTISE_PEER_URLS=http://:2380 +Environment=ETCD_ADVERTISE_CLIENT_URLS=http://:2379 +Environment=ETCD_INITIAL_CLUSTER="=http://:2380,node182=http://:2380" +Environment=ETCD_INITIAL_CLUSTER_STATE="existing" +Environment=ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster" +ExecStart=/usr/local/bin/etcd +Restart=on-failure +RestartSec=5 +User=root + + +[Install] +WantedBy=multi-user.target +``` + +##### 1.2.2.5 在新节点的机器上启动服务 + +```bash +systemctl stop etcd +rm -rf /var/lib/etcd +systemctl daemon-reload +systemctl start etcd +``` + +##### 1.2.2.6 查看集群 + +```bash +etcdctl --endpoints=http://:2379 member list --write-out=table +``` + +##### 1.2.2.7 重复3-6步骤添加新节点进集群 + +### 1.3 其他 + +#### 1.3.1 删除节点 + +查看当前集群成员列表 + +```bash +etcdctl member list +``` + +删除指定ID的节点 + +```bash +etcdctl member remove +``` + +#### 1.3.2 查看etcd集群的Leader + +```bash +etcdctl get mooncake-store/mooncake_cluster/master_view --print-value-only +``` + + + +## 2 MinIO（S3）环境搭建 +### 2.1 下载 MinIO 二进制文件 + +```bash +wget https://dl.min.io/server/minio/release/linux-amd64/minio +``` + +### 2.2 赋予执行权限 + +```bash +chmod +x minio +``` + +### 2.3 移动到系统路径 + +```bash +sudo mv minio /usr/local/bin/ +``` + +### 2.4 创建数据目录 + +```bash +mkdir -p /home/minio_data +``` + +### 2.5 启动MinIO + +设置管理员用户名和密码 + +```bash +export MINIO_ROOT_USER=admin +export MINIO_ROOT_PASSWORD=adminadmin +``` + +配置中写好所有节点的地址，在所有设备中运行 + +```bash +minio server \ + --address ":9000" \ + --console-address ":9001" \ + http:///home/minio_data \ + http:///home/minio_data +``` + + +### 2.6 配置自启动服务(可选) + +创建 `/etc/systemd/system/minio.service`文件： + +```ini +[Unit] +Description=MinIO Object Storage +Documentation=https://docs.min.io +Wants=network-online.target +After=network-online.target + +[Service] +User=root +Group=root +# 直接设置环境变量，不依赖外部文件 +Environment="MINIO_ROOT_USER=admin" +Environment="MINIO_ROOT_PASSWORD=admin12345" +Environment="MINIO_VOLUMES=http:///home/minio_data http:///home/minio_data" +ExecStart=/usr/local/bin/minio server --address ":9000" --console-address ":9001" $MINIO_VOLUMES +Restart=always +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target +``` + +保存后执行： + +```bash +sudo systemctl daemon-reload +sudo systemctl start minio +sudo systemctl status minio +``` + +### 2.8 下载MC + +```bash +# 下载 mc 客户端 +wget https://dl.min.io/client/mc/release/linux-arm64/mc + +# 赋予执行权限 +chmod +x mc + +# 移动到系统路径，方便全局使用 +sudo mv mc /usr/local/bin/ +``` + +### 2.9 连接MinIO服务器并添加别名 + +```bash +mc alias set <别名> http://<你的服务器IP>:9000 <管理员用户名> <密码> +``` + +### 2.10 查看服务器信息 + +```bash +mc admin info myminio +``` + +### 2.11 创建桶 + +```bash +mc mb myminio/mooncake-snapshot +``` + +### 2.12 查看所有桶 + +```bash +mc ls myminio +``` + + +## 3 Mooncake S3环境编译 +### 3.1 配置AWS-S3环境 + +#### 3.1.1 检查环境 + +```bash +ls -l /usr/local/lib64/libaws-cpp-sdk-s3.so +``` + +或 + +```bash +ls -l /usr/local/lib/libaws-cpp-sdk-s3.so +``` + +若文件存在，则环境搭建完成 + +#### 3.1.2 安装（AWS SDK C++） + +```bash +git clone --depth 1 --recurse-submodules --shallow-submodules https://github.com/aws/aws-sdk-cpp.git +cd aws-sdk-cpp +mkdir build && cd build +cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3" -DENABLE_TESTING=OFF .. +make -j$(nproc) +make install +ldconfig +``` + +### 3.2 Mooncake安装 + +```bash +git config --global http.sslVerify false +git clone --recurse-submodules https://github.com/<对应仓>/Mooncake.git +cd Mooncake +git checkout HA +export GOPROXY="http://mirrors.aliyun.com/goproxy,direct" +export GOINSECURE="go.etcd.io/etcd" +export GOSUMDB="sum.golang.org" +export CXXFLAGS="${CXXFLAGS} -w" +export CFLAGS="${CFLAGS} -w" +cmake -B build -DUSE_UB=ON -DBUILD_SHARED_LIBS=ON -DWITH_TE=ON -DWITH_STORE=ON -DWITH_P2P_STORE=ON -DUSE_ETCD=ON -DCMAKE_BUILD_TYPE=Release -DBUILD_UNIT_TESTS:BOOL=OFF -DSTORE_USE_ETCD=ON -DHAVE_AWS_SDK=ON +cmake --build build -j$(nproc) +git config --global http.sslVerify true +``` + +## 4 Mooncake HA测试脚本 +### 4.1 master运行脚本 + +```bash +export URMA_RPC_ENABLE=0 +export URMA_RPC_DEVICE=bonding_dev_0 +export URMA_RPC_EID_INDEX=0 + +export MOONCAKE_SNAPSHOT_LOCAL_PATH=/home/mooncake_snapshot +export MOONCAKE_AWS_ACCESS_KEY_ID="admin" +export MOONCAKE_AWS_SECRET_ACCESS_KEY="adminadmin" +export MOONCAKE_AWS_REGION="us-east-1" +export MOONCAKE_AWS_BUCKET_NAME="mooncake-snapshot" +export MOONCAKE_AWS_S3_ENDPOINT="http://:9000" +export MOONCAKE_MASTER_SERVICE_SNAPSHOT_TEST_SKIP_CLEANUP=1 +export MC_LOG_DIR="/home/master_log" + +mooncake_master \ + --default_kv_lease_ttl=300000 \ + --default_kv_soft_pin_ttl=300000 \ + --metrics_port=9006 \ + --rpc_port=50052 \ + --ha_backend_type=etcd \ + --etcd_endpoints="etcd://:2379;:2379" \ + --enable_ha=true \ + --enable_oplog=true \ + --rpc_address= \ + --enable_metrics_report_to_backend=true \ + --enable-offload=true \ + --enable_snapshot=false \ + --enable_snapshot_restore=false \ + --snapshot_interval_seconds=10 \ + --snapshot_retention_count=5 \ + --snapshot_object_store_type="s3" \ + --v=0 \ + --cluster_id=mooncake_cluster + +``` + +### 4.2 client运行脚本 + +```bash +export MC_URMA_ACTIVE_PORT=0 +export MOONCAKE_OFFLOAD_FILE_STORAGE_PATH=/home/mooncake_ssd +export MC_STORE_CLIENT_METRIC=0 +export MC_STORE_CLIENT_METRIC_INTERVAL=3 +export MC_URMA_BONDING_MULTIPATH_ENABLE=on +export MC_HIFREQ_LOG_SAMPLE_RATE=1 +export MOONCAKE_OFFLOAD_BUCKET_KEYS_LIMIT=1 +export GLOG_v=1 +export MC_LOG_DIR="/home/client_log" + +export URMA_RPC_ENABLE=0 +export URMA_RPC_DEVICE=bonding_dev_0 +export URMA_RPC_EID_INDEX=0 + +mooncake_client \ + --host= \ + --metadata_server="etcd://:2379;:2379" \ + --master_server_address="etcd://:2379;:2379" \ + --protocol=ub \ + --device_names=bonding_dev_0 \ + --global_segment_size=214748364080 \ + --port=50053 \ + --threads=32 \ + --v=0 \ + --enable_offload=true +``` + +### 4.3 数据写入脚本 + +```bash +export MC_STORE_CLIENT_SETUP_RETRIES=3 +export no_proxy="127.0.0.1,localhost,local,.local,192.168.0.0/16,10.0.0.0/8,172.16.0.0/12,141.61.0.0/16" +export MC_STORE_CLIENT_METRIC_BANDWIDTH=0 +export MC_TCP_BIND_ADDRESS= +export MC_URMA_TRANS_MODE=RM + +export MC_URMA_BONDING_MULTIPATH_ENABLE=on +export MC_HIFREQ_LOG_SAMPLE_RATE=1 +export MC_URMA_ACTIVE_PORT=0 + +export URMA_RPC_ENABLE=0 +export URMA_RPC_DEVICE=bonding_dev_0 +export URMA_RPC_EID_INDEX=0 + +export MC_LOG_DIR="/home/w00889253/client_log" + +stress_cluster_bench \ + --metadata-server='etcd://:2379;:2379' \ + --master-server='etcd://:2379;:2379' \ + --local-hostname=$MC_TCP_BIND_ADDRESS \ + --global-segment-size=0 \ + --local-buffer-size=10737418240 \ + --device-name=bonding_dev_0 \ + --scenario=remote_memory \ # 四个场景segment_write,segment_read, remote_memory writer/reader + --role=writer \ + --num-keys=10000 \ + --protocol=ub \ + --verify=true \ + --num_threads=1 \ + --batch-size=16 \ + --duration=0 \ + --master_admin_port=9006 \ + --segments="," \ + --replica_num=1 + + + +``` + +### 4.4 数据读取脚本 + +```bash +export MC_STORE_CLIENT_SETUP_RETRIES=3 +export no_proxy="127.0.0.1,localhost,local,.local,192.168.0.0/16,10.0.0.0/8,172.16.0.0/12,141.61.0.0/16" +export MC_STORE_CLIENT_METRIC_BANDWIDTH=0 +export MC_TCP_BIND_ADDRESS= +export MC_URMA_TRANS_MODE=RM + +export MC_URMA_BONDING_MULTIPATH_ENABLE=on +export MC_HIFREQ_LOG_SAMPLE_RATE=1 +export MC_URMA_ACTIVE_PORT=0 + +export URMA_RPC_ENABLE=0 +export URMA_RPC_DEVICE=bonding_dev_0 +export URMA_RPC_EID_INDEX=0 + +export MC_LOG_DIR="/home/w00889253/client_log" + +stress_cluster_bench \ + --metadata-server='etcd://:2379;:2379' \ + --master-server='etcd://:2379;:2379' \ + --local-hostname=$MC_TCP_BIND_ADDRESS \ + --global-segment-size=0 \ + --local-buffer-size=10737418240 \ + --device-name=bonding_dev_0 \ + --scenario=remote_memory \ + --role=reader \ + --num-keys=10000 \ + --protocol=ub \ + --verify=true \ + --num_threads=1 \ + --batch-size=16 \ + --duration=0 \ + --master_admin_port=9006 \ + --segments="," \ + --replica_num=1 + + +``` + + + + +## 5 多副本参数配置 +### 配置说明：replica_num / nof_replica_num 与 client TTL + +> 代码基线：kvcache-ai/Mooncake `main` +> 内容：① 多副本参数 `replica_num` / `nof_replica_num` 的设置位置与方式；② client TTL 的两种配置途径（摘自《故障感知延迟分析-master多久发现节点故障.md》） + +--- + +### 5.1 replica_num 与 nof_replica_num + +#### 5.1.1 定义 + +定义在 `mooncake-store/include/replica.h` L81-83 的 `ReplicateConfig` 结构体： + +```cpp +struct ReplicateConfig { + size_t replica_num{1}; // 内存副本数，默认 1 + size_t nof_replica_num{0}; // NoF（NVMe-oF SSD）副本数，默认 0 + ... +}; +``` + +#### 5.1.2 设置方式（三种场景） + +**方式 1：C++ 代码中直接赋值** + +```cpp +ReplicateConfig config; +config.replica_num = 2; // 2 个内存副本 +config.nof_replica_num = 1; // 1 个 NoF 副本 +auto result = client->Put(key, slices, config); +``` + +参考：`mooncake-store/tests/task_integration_test.cpp`、`mooncake-store/tests/replica_count_verify_test.cpp`。 + +**方式 2：Python API** + +通过 pybind11 暴露为可读写属性（`mooncake-integration/store/store_py.cpp` L1832-1833，`def_readwrite`）： + +```python +from mooncake.store import ReplicateConfig + +config = ReplicateConfig() +config.replica_num = 1 # 1 个内存副本 +config.nof_replica_num = 1 # 1 个 NoF 副本 +rc = store.put(key, data, config) +``` + +参考：`mooncake-wheel/tests/test_replicated_distributed_object_store.py`、`scripts/test_copy_move_api.py`。 + +**方式 3：命令行参数 / 环境变量（E2E 与 Benchmark）** + +| 场景 | 参数 | 默认值 | +|---|---|---| +| E2E client（`store_client_e2e.py`） | `--memory-replica-num` / `--nof-replica-num` | 1 / 1 | +| E2E 脚本（`run_nof_heartbeat_tcp_e2e.sh`） | 环境变量 `CLIENT_MEMORY_REPLICA_NUM` / `CLIENT_NOF_REPLICA_NUM` | 1 / 1 | +| Benchmark（`store_kv_bench.py`） | `--memory-replica-num` / `--nof-replica-num` | 1 / 0 | + +E2E 用法示例： + +```bash +CLIENT_MEMORY_REPLICA_NUM=1 CLIENT_NOF_REPLICA_NUM=1 bash run_nof_heartbeat_tcp_e2e.sh +``` + +#### 5.1.3 这两个值如何决定副本写入模式 + +`DetermineReplicaWriteMode`（`mooncake-store/include/replica.h` L156-165）： + +| `replica_num` | `nof_replica_num` | 写入模式 | 语义 | +|---|---|---|---| +| 1 | 0 | `SINGLE_REPLICA` | 单副本 | +| 1 | 1 | `FLEXIBLE_DUAL_REPLICA` | 灵活双副本：1 内存 + 1 NoF，best-effort，任一类型成功即可（`HasExpectedReplicaAllocation` 只检查 `memory+nof > 0`） | +| >1 或 >1 | 任意 | `RELIABLE_MULTI_REPLICA` | 可靠多副本：必须严格凑齐（`allocated_memory == replica_num` 且 `allocated_nof == nof_replica_num`） | +| 0 | 0 | `SINGLE_REPLICA` | 退化为单副本 | + +#### 5.1.4 关键约束 + +1. **`nof_replica_num > 0` 必须 USE_NOF=ON 编译**：否则 master 的 `PutStart` 直接返回 `INVALID_PARAMS`（`master_service.cpp` L3081-3088，`#ifndef USE_NOF` 分支；测试见 `replica_count_verify_test.cpp` Test 4）。 +2. **`replica_num` 与 `nof_replica_num` 不能同时为 0**：`store_kv_bench.py` L653 有显式校验 `ValueError`。 +3. **best-effort 语义**：`nof_replica_num == 0` 时，`HasExpectedReplicaAllocation`（`master_service.cpp` L121-133）只检查 `allocated_memory > 0`，不要求严格等于 `replica_num`。实测 `replica_num=5` 而只有 4 个 segment 时，分配 4 个副本仍算成功。 + +--- + +### 5.2 client TTL 的两种配置 + +#### 5.2.1 背景结论 + +- **master 多久发现 client 故障**：默认 **9~11 秒**（= client TTL ± 1s，TTL 默认 10s）。机制是"租约到期"，不是"心跳中断检测"。 +- **可配置项**：client TTL 可配置；client 心跳间隔（1s，`client_service.cpp` L3743 硬编码）和 master 检查周期（1s，`master_service.h` L1980 硬编码）不可配。 +- **配置位置**：这是 **`mooncake_master` 的启动参数，不是 client 的**。 + +#### 5.2.2 两种配置途径 + +**途径 1：命令行参数 `--client_ttl`**。gflag 定义（`master.cpp` L260-264），启动时 `mooncake_master --client_ttl=5` 即生效： + +```cpp +DEFINE_int64( + client_ttl, mooncake::DEFAULT_CLIENT_LIVE_TTL_SEC, + "Seconds a client stays considered alive after the last heartbeat. " + "If this TTL elapses without a refresh, the master treats the " + "client as disconnected and may unmount its segments"); +``` + +**途径 2：master 配置文件（键名 `client_live_ttl_sec`）**。master 启动时若指定了配置文件，会从中读这个键；读不到则用 gflag 的值兜底（`master.cpp` L465-467，第三个参数就是兜底值 `FLAGS_client_ttl`）： + +```cpp + default_config.GetInt64("client_live_ttl_sec", + &master_config.client_live_ttl_sec, + FLAGS_client_ttl); +``` + +**优先级：命令行显式设置 > 配置文件**。命令行显式传了 `--client_ttl`（`!info.is_default` 判断非默认值），或根本没用配置文件（`!conf_set`），则用命令行值覆盖（`master.cpp` L949-953）： + +```cpp + if ((google::GetCommandLineFlagInfo("client_ttl", &info) && + !info.is_default) || + !conf_set) { + master_config.client_live_ttl_sec = FLAGS_client_ttl; + } +``` + +#### 5.2.3 配置速查表 + +| 参数 | 默认 | 可配 | 配置在哪 | 源码位置 | +|---|---|---|---|---| +| client TTL | 10s | ✅ | `mooncake_master --client_ttl=<秒>` / master 配置文件 `client_live_ttl_sec` | `master.cpp` L260-264 / L465-467 / L949-953；`types.h` L95 `DEFAULT_CLIENT_LIVE_TTL_SEC=10` | +| client 心跳间隔 | 1s | ❌ | —（client 代码硬编码） | `client_service.cpp` L3743 | +| master 检查周期 | 1s | ❌ | —（master 代码硬编码） | `master_service.h` L1980 | + +一句话：同一个参数，命令行叫 `client_ttl`、配置文件里叫 `client_live_ttl_sec`，最终都写进 `master_config.client_live_ttl_sec`，两头都设时命令行优先。 + +实测：E2E 中 master 以 `--client_ttl=5` 启动，kill 数据节点后感知延迟落在 4~6s 区间，与理论吻合。 + +### 5.3 bench方案验证 + +验证思路：验证内存存在副本和disk磁盘存在副本两种方式进行验证。 + +#### 5.3.1 内存存在副本验证思路 + +写入500个key，如果存在两个client，则应该两边分别写入250个key，同时应该在另外一个client上面存在副本，则每个节点client内存应该有500个key。 +操作步骤：先写入500个key，然后断连一个节点client，读取全部的key，可以从另外一个节点上全部读取。 + +#### 5.3.2 disk磁盘存在副本验证思路 + +写入500个key，如果存在两个client，则两边都存在250个key，，同时应该在另外一个client上面存在副本，则每个节点client内存应该有500个key。同时将这些key卸载到本地磁盘当中，磁盘当中应该在每个节点上有完整的500个key。 + +操作步骤：先写入500个key，然后断连两个节点client， 重启其中一个client，此时磁盘中的卸载的数据可以在master上重新加载，有500个key，此时读取数据应该能够将所有的数据都读到。 + +参考脚本： +master + +``` +mooncake_master \ + --enable_http_metadata_server=true \ + --http_metadata_server_host=0.0.0.0 \ + --http_metadata_server_port=9109 \ + --default_kv_lease_ttl=300000 \ + --default_kv_soft_pin_ttl=300000 \ + --metrics_port=9006 \ + --rpc_port=50052 \ + --enable-offload=true + +``` + +client + +``` +export MC_TCP_BIND_ADDRESS= +export MC_URMA_ACTIVE_PORT=0 +export MOONCAKE_OFFLOAD_FILE_STORAGE_PATH=/home/mooncake_ssd +export MC_STORE_CLIENT_METRIC=0 +export MC_STORE_CLIENT_METRIC_INTERVAL=3 +export MC_URMA_BONDING_MULTIPATH_ENABLE=on +export MC_HIFREQ_LOG_SAMPLE_RATE=0 +export MOONCAKE_OFFLOAD_BUCKET_KEYS_LIMIT=1(重要，确保所有key存入到磁盘当中） + +export URMA_RPC_ENABLE=0 +export URMA_RPC_DEVICE=bonding_dev_0 +export URMA_RPC_EID_INDEX=0 +mooncake_client \ + --host= \ + --metadata_server=http://:9109/metadata \ + --master_server_address=:50052 \ + --protocol=ub \ + --device_names=bonding_dev_0 \ + --global_segment_size=21474836408 \ + --port=50053 \ + --threads=32 \ + --v=0 \ + --enable_offload=true + +``` + +写入/读取脚本 + +``` +export MC_STORE_CLIENT_SETUP_RETRIES=3 +export no_proxy="127.0.0.1,localhost,local,.local,192.168.0.0/16,10.0.0.0/8,172.16.0.0/12,141.61.0.0/16" +export MC_STORE_CLIENT_METRIC_BANDWIDTH=0 +export MC_TCP_BIND_ADDRESS= +export MC_URMA_TRANS_MODE=RM +export MC_LOG_ENABLE=off +# export MC_LOG_DIR=/home/log +export MC_URMA_BONDING_MULTIPATH_ENABLE=on +export MC_HIFREQ_LOG_SAMPLE_RATE=0 +export MC_URMA_ACTIVE_PORT=0 + +export URMA_RPC_ENABLE=0 +export URMA_RPC_DEVICE=bonding_dev_0 +export URMA_RPC_EID_INDEX=0 + +stress_cluster_bench \ + --metadata-server='http://:9109/metadata' \ + --master-server=':50052' \ + --local-hostname=$MC_TCP_BIND_ADDRESS \ + --global-segment-size=0 \ + --local-buffer-size=10737418240 \ + --device-name=bonding_dev_0 \ + --scenario=remote_memory \ （使用remote memory，这样key可以以固定结构写入） + --role=reader \ （写入数据使用writer，读取使用reader） + --num-keys=500 \ + --protocol=ub \ + --verify=false \ + --num_threads=1 \ + --batch-size=16 \ + --duration=0 \ + --master_admin_port=9006 \ + --replica_num=2 （副本数量） + +``` + + +# 6 mooncake基础信息上报 + +在上述启动master的脚本中，配置```enable_metrics_report_to_backend=true``,可以通过etcd命令行查找到对应数据 +参考示例如下： + +``` +[root@node1 Mooncake]# etcdctl --endpoints=:2379,:2379 get --prefix /mooncake_cluster/masters/primary +/mooncake_cluster/masters/primary +{"id":"4056379308262733915-15491753023959092137","hostname":":50052","role":"primary","mem_total_bytes":42949672816,"mem_used_bytes":4194304000,"mem_available_bytes":38755368816,"nof_total_bytes":0,"nof_used_bytes":0,"nof_available_bytes":0,"file_total_bytes":4398046511104,"file_used_bytes":9080668160,"file_available_bytes":4388965842944,"key_count":500,"active_clients":2,"updated_at":"2026-08-06T16:59:35+08:00"} +[root@node1 Mooncake]# etcdctl --endpoints=:2379,:2379 get --prefix /mooncake_cluster/masters/primary +/mooncake_cluster/masters/primary +{"id":"4056379308262733915-15491753023959092137","hostname":":50052","role":"primary","mem_total_bytes":42949672816,"mem_used_bytes":4194304000,"mem_available_bytes":38755368816,"nof_total_bytes":0,"nof_used_bytes":0,"nof_available_bytes":0,"file_total_bytes":4398046511104,"file_used_bytes":9080668160,"file_available_bytes":4388965842944,"key_count":500,"active_clients":2,"updated_at":"2026-08-06T17:00:40+08:00"} +``` + + +# 7 mooncake性能信息 + +使用脚本```Mooncake/mooncake-store/benchmarks/cluster_mooncake_diag.py``` +参考其中的使用方法，在对应脚本当中设置对应log日志，则可以进行读取。 \ No newline at end of file diff --git a/mooncake-common/etcd/etcd_wrapper.go b/mooncake-common/etcd/etcd_wrapper.go index 7cedada048..f75a1e3415 100644 --- a/mooncake-common/etcd/etcd_wrapper.go +++ b/mooncake-common/etcd/etcd_wrapper.go @@ -811,8 +811,49 @@ func EtcdStoreBatchCreateWrapper(keys **C.char, values **C.char, count C.int, er return 0 } +//export EtcdStoreBatchPutWithLeaseWrapper +func EtcdStoreBatchPutWithLeaseWrapper(keys **C.char, keySizes *C.int, values **C.char, valueSizes *C.int, count C.int, leaseId int64, errMsg **C.char) int { + cli := getStoreClient() + if cli == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + + n := int(count) + if n == 0 { + return 0 + } + + // Unsafe casting to access C arrays as Go slices; key/value buffers are + // binary-safe (carry explicit sizes). + keyPtrs := (*[1 << 28]*C.char)(unsafe.Pointer(keys))[:n:n] + keySizeList := (*[1 << 28]C.int)(unsafe.Pointer(keySizes))[:n:n] + valPtrs := (*[1 << 28]*C.char)(unsafe.Pointer(values))[:n:n] + valSizeList := (*[1 << 28]C.int)(unsafe.Pointer(valueSizes))[:n:n] + + ops := make([]clientv3.Op, 0, n) + for i := 0; i < n; i++ { + k := C.GoStringN(keyPtrs[i], keySizeList[i]) + v := C.GoStringN(valPtrs[i], valSizeList[i]) + // Bind all keys to the single master lease. Caller is responsible for + // fencing (the keys must already be absent / owned by self); this is an + // unconditional atomic batch, so it can also be reused for reaffirm. + ops = append(ops, clientv3.OpPut(k, v, clientv3.WithLease(clientv3.LeaseID(leaseId)))) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := cli.Txn(ctx).Then(ops...).Commit() + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + return 0 +} + //export EtcdStoreTxnCompareAndPutWrapper -func EtcdStoreTxnCompareAndPutWrapper(compareKeys **C.char, compareKeySizes *C.int, compareKinds *C.int, compareValues **C.char, compareValueSizes *C.int, compareCount C.int, putKeys **C.char, putKeySizes *C.int, putValues **C.char, putValueSizes *C.int, putCount C.int, errMsg **C.char) int { +func EtcdStoreTxnCompareAndPutWrapper(compareKeys **C.char, compareKeySizes *C.int, compareKinds *C.int, compareValues **C.char, compareValueSizes *C.int, compareCount C.int, putKeys **C.char, putKeySizes *C.int, putValues **C.char, putValueSizes *C.int, putCount C.int, deleteKeys **C.char, deleteKeySizes *C.int, deleteCount C.int, errMsg **C.char) int { cli := getStoreClient() if cli == nil { *errMsg = C.CString("etcd client not initialized") @@ -821,6 +862,7 @@ func EtcdStoreTxnCompareAndPutWrapper(compareKeys **C.char, compareKeySizes *C.i cmpN := int(compareCount) putN := int(putCount) + deleteN := int(deleteCount) cmps := make([]clientv3.Cmp, 0, cmpN) if cmpN > 0 { @@ -844,7 +886,7 @@ func EtcdStoreTxnCompareAndPutWrapper(compareKeys **C.char, compareKeySizes *C.i } } - ops := make([]clientv3.Op, 0, putN) + ops := make([]clientv3.Op, 0, putN+deleteN) if putN > 0 { putKeyPtrs := (*[1 << 28]*C.char)(unsafe.Pointer(putKeys))[:putN:putN] putKeySizeList := (*[1 << 28]C.int)(unsafe.Pointer(putKeySizes))[:putN:putN] @@ -856,6 +898,14 @@ func EtcdStoreTxnCompareAndPutWrapper(compareKeys **C.char, compareKeySizes *C.i ops = append(ops, clientv3.OpPut(k, v)) } } + if deleteN > 0 { + deleteKeyPtrs := (*[1 << 28]*C.char)(unsafe.Pointer(deleteKeys))[:deleteN:deleteN] + deleteKeySizeList := (*[1 << 28]C.int)(unsafe.Pointer(deleteKeySizes))[:deleteN:deleteN] + for i := 0; i < deleteN; i++ { + k := C.GoStringN(deleteKeyPtrs[i], deleteKeySizeList[i]) + ops = append(ops, clientv3.OpDelete(k)) + } + } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() diff --git a/mooncake-common/include/rpc_client_io_context.h b/mooncake-common/include/rpc_client_io_context.h index b9842f0c99..fb575df939 100644 --- a/mooncake-common/include/rpc_client_io_context.h +++ b/mooncake-common/include/rpc_client_io_context.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -26,9 +27,10 @@ coro_io::io_context_pool& GetRpcClientIoContextPool(uint32_t thread_count) { } /** - * A replaceable client pool for callers that communicate with one target at a - * time. Requests retain a shared_ptr to the old pool while they are in flight; - * after an address switch the old pool is destroyed when those requests end. + * A client pool accessor that caches one client pool per target address, so + * callers that alternate between several targets (e.g. submaster routing) + * reuse existing connections instead of recreating a pool on every switch. + * The most recently selected pool is also exposed via GetClientPool(). */ class RpcClientPool { public: @@ -38,18 +40,22 @@ class RpcClientPool { explicit RpcClientPool(coro_io::io_context_pool& io_context_pool, PoolConfig config = {}) : io_context_pool_(io_context_pool), config_(std::move(config)) { - // Address replacement supersedes background recovery of the old host. + // Explicit target selection supersedes background recovery of an old + // host; the pool's own connect/retry still applies per request. config_.host_alive_detect_duration = std::chrono::seconds(0); } std::shared_ptr GetOrCreateClientPool( std::string_view address) { std::lock_guard lock(mutex_); - if (!client_pool_ || address_ != address) { - client_pool_ = - ClientPool::create(address, config_, io_context_pool_); - address_ = address; + std::string addr(address); + auto it = pools_.find(addr); + if (it == pools_.end()) { + auto pool = ClientPool::create(addr, config_, io_context_pool_); + it = pools_.emplace(addr, std::move(pool)).first; } + address_ = std::move(addr); + client_pool_ = it->second; return client_pool_; } @@ -58,12 +64,21 @@ class RpcClientPool { return client_pool_; } + std::string GetAddress() const { + std::shared_lock lock(mutex_); + return address_; + } + private: mutable std::shared_mutex mutex_; coro_io::io_context_pool& io_context_pool_; PoolConfig config_; std::string address_; std::shared_ptr client_pool_; + // Cache of client pools keyed by target address, so switching back and + // forth between submaster addresses reuses existing connections instead of + // recreating the pool on every switch. + std::unordered_map> pools_; }; } // namespace mooncake diff --git a/mooncake-common/tests/rpc_client_io_context_test.cpp b/mooncake-common/tests/rpc_client_io_context_test.cpp index c3b18c7127..d63fd5e34e 100644 --- a/mooncake-common/tests/rpc_client_io_context_test.cpp +++ b/mooncake-common/tests/rpc_client_io_context_test.cpp @@ -41,18 +41,19 @@ TEST(RpcClientIoContextPoolTest, UsesConfiguredSizeAndReusesPool) { EXPECT_NE(&first_pool, &second_pool); } -TEST(RpcClientIoContextPoolTest, ReplacesPoolWhenTargetChanges) { +TEST(RpcClientIoContextPoolTest, CachesPoolsPerAddress) { RpcClientPool pools(GetFirstTestRpcClientIoContextPool()); auto first = pools.GetOrCreateClientPool("127.0.0.1:10001"); - std::weak_ptr old_pool = first; EXPECT_EQ(pools.GetOrCreateClientPool("127.0.0.1:10001"), first); auto second = pools.GetOrCreateClientPool("127.0.0.1:10002"); EXPECT_NE(first, second); - first.reset(); - EXPECT_TRUE(old_pool.expired()); EXPECT_EQ(pools.GetClientPool(), second); + + // Switching back reuses the cached pool instead of recreating it. + EXPECT_EQ(pools.GetOrCreateClientPool("127.0.0.1:10001"), first); + EXPECT_EQ(pools.GetClientPool(), first); } TEST(RpcClientIoContextPoolTest, SendsToNewAddressAfterSwitch) { diff --git a/mooncake-store/benchmarks/CMakeLists.txt b/mooncake-store/benchmarks/CMakeLists.txt index 003f0b8724..422d911362 100644 --- a/mooncake-store/benchmarks/CMakeLists.txt +++ b/mooncake-store/benchmarks/CMakeLists.txt @@ -65,3 +65,10 @@ if(STORE_USE_ETCD) oplog_batch_bench PRIVATE mooncake_store glog::glog gflags::gflags JsonCpp::JsonCpp) endif() + +if(STORE_USE_ETCD) + add_executable(vchunk_distributed_bench vchunk_distributed_bench.cpp) + target_link_libraries( + vchunk_distributed_bench PRIVATE mooncake_store transfer_engine asio_shared + gflags::gflags glog::glog pthread) +endif() diff --git a/mooncake-store/benchmarks/stress_cluster_bench.cpp b/mooncake-store/benchmarks/stress_cluster_bench.cpp index 5da134b429..fcd4e445ce 100644 --- a/mooncake-store/benchmarks/stress_cluster_bench.cpp +++ b/mooncake-store/benchmarks/stress_cluster_bench.cpp @@ -190,7 +190,7 @@ DEFINE_string(ssd_offload_path, "", "SSD offload directory path"); DEFINE_string(scenario, "local_memory", "Benchmark scenario: local_memory, remote_memory, local_disk, " - "remote_disk, segment_write, segment_read"); + "remote_disk, segment_write, segment_read, remove, batch_remove"); DEFINE_string(role, "writer", "Node role: writer (prefill data) or reader (benchmark reads)"); DEFINE_uint64(value_size, 4 * MB, "Size of each value in bytes"); @@ -719,6 +719,21 @@ class StressBenchmark { return "seg_" + sanitized + "_key_" + std::to_string(idx); } + // Key generator for remove/batch_remove scenarios. Uses a distinct + // "rmv_" prefix so remove keys never collide with segment_write keys + // ("seg_"). This allows remove and write benchmarks to run independently + // without cross-contamination. + static std::string MakeRemoveKey(const std::string& segment, size_t idx) { + static const char* kSpecialChars = ".:-/\\[]{}()@#$%^&*+=|<>,;!?`'\"~"; + std::string sanitized = segment; + for (char& c : sanitized) { + if (std::strchr(kSpecialChars, c) != nullptr || std::isspace(c)) { + c = '_'; + } + } + return "rmv_" + sanitized + "_key_" + std::to_string(idx); + } + int RunSegmentWrite() { auto segments = DiscoverSegmentsIfNeeded( "--segments not specified, auto-discovering"); @@ -1253,6 +1268,113 @@ class StressBenchmark { return 0; } + int RunSegmentRemove(bool use_batch) { + auto segments = DiscoverSegmentsIfNeeded( + "--segments not specified, auto-discovering"); + if (segments.empty()) { + return -1; + } + LOG(INFO) << "Discovered " << segments.size() + << " segments from master"; + + size_t remove_segment_nums = FLAGS_read_segment_nums; + if (remove_segment_nums == 0 || + remove_segment_nums > segments.size()) { + remove_segment_nums = segments.size(); + } + std::vector remove_segments( + segments.begin(), segments.begin() + remove_segment_nums); + + LOG(INFO) << "=== SEGMENT REMOVE MODE ===" + << (use_batch ? " (batch)" : " (single key)"); + LOG(INFO) << "Removing from " << remove_segment_nums << " segments (" + << remove_segment_nums << " nodes)"; + for (size_t s = 0; s < remove_segments.size(); ++s) { + LOG(INFO) << " Segment [" << s << "]: " << remove_segments[s]; + } + LOG(INFO) << "Keys per segment: " << FLAGS_num_keys; + LOG(INFO) << "Batch size: " << FLAGS_batch_size; + + if (FLAGS_duration > 0) { + LOG(WARNING) << "--duration is ignored for remove scenarios: " + << "removal is not idempotent, a single pass is used"; + } + + // Phase 1: prefill each segment. Key layout and preferred_segments + // pinning mirror RunSegmentWrite exactly. + std::vector configs(remove_segments.size()); + for (size_t s = 0; s < remove_segments.size(); ++s) { + configs[s].replica_num = FLAGS_replica_num; + configs[s].with_hard_pin = FLAGS_hard_pin; + configs[s].preferred_segments = {remove_segments[s]}; + } + + LOG(INFO) << "Phase 1: Prefilling " << FLAGS_num_keys + << " keys to " << remove_segment_nums + << " segments (interleaved), each " + << FLAGS_value_size / MB << " MB"; + for (size_t i = 0; i < FLAGS_num_keys; ++i) { + for (size_t s = 0; s < remove_segments.size(); ++s) { + const auto& segment = remove_segments[s]; + std::string key = MakeRemoveKey(segment, i); + FillBuffer(i); + int ret = client_->put_from(key, buffer_, FLAGS_value_size, + configs[s]); + if (ret != 0) { + LOG(ERROR) << "put_from failed for key=" << key + << " segment=" << segment << " ret=" << ret; + return ret; + } + } + if ((i + 1) % 10 == 0 || i == FLAGS_num_keys - 1) { + LOG(INFO) << " Prefilled " << (i + 1) << "/" << FLAGS_num_keys + << " keys to all " << remove_segment_nums + << " segments"; + } + } + LOG(INFO) << "Prefill phase complete"; + + // Phase 2: assemble the full key list in the same order as + // RunSegmentRead (outer key index, inner segment). + std::vector all_keys; + for (size_t i = 0; i < FLAGS_num_keys; ++i) { + for (size_t s = 0; s < remove_segments.size(); ++s) { + all_keys.push_back(MakeRemoveKey(remove_segments[s], i)); + } + } + LOG(INFO) << "Total keys to remove: " << all_keys.size(); + + // Phase 3: concurrent remove (single pass, mirrors segment_read). + LOG(INFO) << "Phase 3: Concurrent " << (use_batch ? "batch " : "") + << "remove with " << FLAGS_num_threads << " threads"; + + BenchmarkStats stats; + stats.InitThreads(FLAGS_num_threads, + all_keys.size() / FLAGS_num_threads); + stats.StartTimer(); + + std::latch start_latch(static_cast(FLAGS_num_threads)); + std::latch done_latch(static_cast(FLAGS_num_threads)); + auto threads = LaunchRemoveWorkers( + FLAGS_num_threads, all_keys.size(), stats, start_latch, done_latch, + use_batch, [&all_keys](size_t idx) { + return all_keys[idx % all_keys.size()]; + }); + + done_latch.wait(); + stats.StopTimer(); + + for (auto& th : threads) { + th.join(); + } + + stats.Finalize(); + stats.Print(use_batch ? "SEGMENT BATCH REMOVE BENCHMARK" + : "SEGMENT REMOVE BENCHMARK"); + + return 0; + } + int RunListSegments() { LOG(INFO) << "Discovering segments from master at " << FLAGS_master_server << ":" << FLAGS_master_admin_port; @@ -1300,6 +1422,10 @@ class StressBenchmark { return RunSegmentRead(); } else if (FLAGS_scenario == "list_segments") { return RunListSegments(); + } else if (FLAGS_scenario == "remove") { + return RunSegmentRemove(false); + } else if (FLAGS_scenario == "batch_remove") { + return RunSegmentRemove(true); } else if (FLAGS_scenario == "remote_memory" || FLAGS_scenario == "remote_disk") { if (FLAGS_role == "writer") { @@ -1473,6 +1599,108 @@ class StressBenchmark { return threads; } + void RemoveWorker(size_t tid, size_t my_keys, size_t key_offset, + BenchmarkStats& stats, std::latch& start_latch, + std::latch& done_latch, bool use_batch, + const std::function& key_func) { + bindToSocket(tid % NR_SOCKETS); + + ThreadResult& result = stats.GetThreadResult(tid); + result.latencies_ns.reserve(my_keys); + + start_latch.arrive_and_wait(); + + size_t keys = 0; + size_t queries = 0; + size_t failed = 0; + size_t bytes = 0; + + if (!use_batch) { + for (size_t i = 0; i < my_keys; ++i) { + size_t key_idx = key_offset + i; + std::string key = key_func(key_idx); + + auto t0 = Clock::now(); + int ret = client_->remove(key, /*force=*/true); + auto t1 = Clock::now(); + + int64_t lat_ns = ElapsedNanos(t0, t1); + result.latencies_ns.push_back(lat_ns); + + if (ret != 0) { + ++failed; + LOG_EVERY_N(ERROR, 100) + << "remove failed key=" << key << " ret=" << ret; + } else { + // Account removed payload as transferred bytes so throughput + // stats stay comparable with the read benchmark. + bytes += FLAGS_value_size; + } + ++keys; + ++queries; + } + } else { + size_t per_key_buf = FLAGS_value_size; + size_t i = 0; + while (i < my_keys) { + std::vector key_list; + size_t batch_end = std::min(i + FLAGS_batch_size, my_keys); + key_list.reserve(batch_end - i); + + for (size_t j = i; j < batch_end; ++j) { + size_t key_idx = key_offset + j; + key_list.push_back(key_func(key_idx)); + } + + auto t0 = Clock::now(); + auto results = client_->batchRemove(key_list, /*force=*/true); + auto t1 = Clock::now(); + + int64_t lat_ns = ElapsedNanos(t0, t1); + result.latencies_ns.push_back(lat_ns); + + for (size_t k = 0; k < results.size(); ++k) { + if (results[k] != 0) { + ++failed; + } else { + bytes += per_key_buf; + } + ++keys; + } + ++queries; + + i = batch_end; + } + } + + result.total_bytes = bytes; + result.total_keys = keys; + result.total_queries = queries; + result.failed_ops = failed; + + done_latch.arrive_and_wait(); + } + + std::vector LaunchRemoveWorkers( + size_t num_threads, size_t total_keys, BenchmarkStats& stats, + std::latch& start_latch, std::latch& done_latch, bool use_batch, + const std::function& key_func) { + std::vector threads; + size_t keys_per_thread = total_keys / num_threads; + size_t remainder = total_keys % num_threads; + + for (size_t t = 0; t < num_threads; ++t) { + size_t my_keys = keys_per_thread + (t < remainder ? 1 : 0); + size_t key_offset = t * keys_per_thread + std::min(t, remainder); + + threads.emplace_back([&, t, my_keys, key_offset, use_batch]() { + RemoveWorker(t, my_keys, key_offset, stats, start_latch, + done_latch, use_batch, key_func); + }); + } + return threads; + } + std::vector DiscoverSegmentsIfNeeded( const std::string& context) { auto segments = ParseSegments(); diff --git a/mooncake-store/benchmarks/vchunk_distributed_bench.cpp b/mooncake-store/benchmarks/vchunk_distributed_bench.cpp new file mode 100644 index 0000000000..712684ce27 --- /dev/null +++ b/mooncake-store/benchmarks/vchunk_distributed_bench.cpp @@ -0,0 +1,229 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "master_client.h" +#include "vchunk_client.h" +#include "vchunk_control_plane.h" +#include "vchunk_transfer_engine.h" + +DEFINE_string(master, "127.0.0.1:50051", "Master RPC address"); +DEFINE_string(routing_etcd, "", + "ETCD endpoints used to load the CVM SubMaster route snapshot"); +DEFINE_string(cluster_namespace, "", + "CVM cluster namespace; required with --routing_etcd"); +DEFINE_string(metadata, "http://127.0.0.1:8080/metadata", + "TransferEngine metadata connection string"); +DEFINE_string(local_server, "vchunk-bench-client:12345", + "Unique TransferEngine client endpoint"); +DEFINE_string(protocol, "tcp", "TransferEngine transport protocol"); +DEFINE_bool(auto_discovery, false, + "Enable TransferEngine device auto-discovery (normally required " + "for RDMA without an explicit device matrix)"); +DEFINE_string(tenant, "vchunk-benchmark", "Tenant id"); +DEFINE_uint64(object_size, 1048576, "Object size in bytes"); +DEFINE_uint64(operations, 1000, "Put/Get/Remove transactions"); +DEFINE_uint32(concurrency, 8, "Concurrent workers"); +DEFINE_uint32(timeout_ms, 30000, "Per-operation timeout"); + +namespace mooncake { +namespace { +using Clock = std::chrono::steady_clock; + +class DisabledLegacyPath final : public VChunkLegacyPath { + public: + ErrorCode Put(const TenantId&, const std::string&, const void*, size_t) + override { + return ErrorCode::INVALID_PARAMS; + } + ErrorCode Get(const TenantId&, const std::string&, void*, size_t) override { + return ErrorCode::INVALID_PARAMS; + } + ErrorCode Remove(const TenantId&, const std::string&) override { + return ErrorCode::INVALID_PARAMS; + } +}; + +double Percentile(std::vector values, double percentile) { + if (values.empty()) return 0; + std::sort(values.begin(), values.end()); + const auto index = static_cast( + percentile * static_cast(values.size() - 1)); + return values[index]; +} + +ErrorCode ConfigureRouting(MasterClient& master) { + if (FLAGS_routing_etcd.empty() && FLAGS_cluster_namespace.empty()) { + return ErrorCode::OK; + } + if (FLAGS_routing_etcd.empty() || FLAGS_cluster_namespace.empty()) { + return ErrorCode::INVALID_PARAMS; + } + return master.LoadRoutingFromEtcd(FLAGS_routing_etcd, + FLAGS_cluster_namespace); +} +} // namespace +} // namespace mooncake + +int main(int argc, char** argv) { + gflags::ParseCommandLineFlags(&argc, &argv, true); + google::InitGoogleLogging(argv[0]); + using namespace mooncake; + if (FLAGS_object_size == 0 || FLAGS_operations == 0 || + FLAGS_concurrency == 0 || FLAGS_timeout_ms == 0) { + LOG(ERROR) << "benchmark numeric arguments must be greater than zero"; + return 2; + } + + MasterClient master(generate_uuid(), nullptr, FLAGS_tenant); + if (const auto error = master.Connect(FLAGS_master); + error != ErrorCode::OK) { + LOG(ERROR) << "failed to connect master: " << toString(error); + return 2; + } + if (const auto error = ConfigureRouting(master); error != ErrorCode::OK) { + LOG(ERROR) << "failed to load SubMaster routing: " << toString(error); + return 2; + } + auto runtime = master.GetVChunkRuntimeInfo(); + if (!runtime || !runtime->enabled || !runtime->persistent_metadata) { + LOG(ERROR) << "master must enable vchunk with persistent ETCD metadata"; + return 2; + } + TransferEngine engine(FLAGS_auto_discovery); + if (engine.init(FLAGS_metadata, FLAGS_local_server) != 0 || + engine.installTransport(FLAGS_protocol, nullptr) == nullptr) { + LOG(ERROR) << "failed to initialize TransferEngine"; + return 2; + } + TransferEngineVChunkDataPlane data_plane(engine); + DisabledLegacyPath legacy; + const TenantId tenant(FLAGS_tenant); + + std::atomic next{0}; + std::atomic succeeded{0}; + std::atomic failed{0}; + std::vector latencies(FLAGS_operations); + std::vector workers; + workers.reserve(FLAGS_concurrency); + std::atomic setup_failed{false}; + std::barrier start_barrier(FLAGS_concurrency + 1); + std::barrier finish_barrier(FLAGS_concurrency + 1); + for (uint32_t worker_id = 0; worker_id < FLAGS_concurrency; ++worker_id) { + workers.emplace_back([&, worker_id] { + MasterClient worker_master(generate_uuid(), nullptr, FLAGS_tenant); + const auto connect_error = worker_master.Connect(FLAGS_master); + const auto routing_error = + connect_error == ErrorCode::OK + ? ConfigureRouting(worker_master) + : connect_error; + RpcVChunkControlPlane control_plane(worker_master); + std::vector source(FLAGS_object_size); + std::vector destination(FLAGS_object_size); + const bool source_registered = + engine.registerLocalMemory(source.data(), source.size(), + "cpu:0") == 0; + const bool destination_registered = + engine.registerLocalMemory(destination.data(), + destination.size(), "cpu:0") == 0; + if (!source_registered || !destination_registered || + routing_error != ErrorCode::OK) { + setup_failed.store(true); + } + start_barrier.arrive_and_wait(); + if (setup_failed.load()) { + finish_barrier.arrive_and_wait(); + if (source_registered) + engine.unregisterLocalMemory(source.data()); + if (destination_registered) + engine.unregisterLocalMemory(destination.data()); + return; + } + VChunkClient client( + true, control_plane, data_plane, legacy, + std::chrono::milliseconds(FLAGS_timeout_ms), + [] { return getCurrentTimeInMilli(); }); + while (true) { + const auto operation = next.fetch_add(1); + if (operation >= FLAGS_operations) break; + for (size_t i = 0; i < source.size(); ++i) { + source[i] = static_cast((operation + i) & 0xff); + } + const auto key = "distributed-" + + std::to_string(worker_id) + "-" + + std::to_string(operation); + const auto operation_started = Clock::now(); + auto error = client.Put(tenant, key, source.data(), + source.size()); + if (error == ErrorCode::OK) { + error = client.Get(tenant, key, destination.data(), + destination.size()); + } + const bool data_matches = destination == source; + const auto remove_error = client.Remove(tenant, key); + latencies[operation] = + std::chrono::duration( + Clock::now() - operation_started) + .count(); + if (error == ErrorCode::OK && data_matches && + remove_error == ErrorCode::OK) { + succeeded.fetch_add(1); + } else { + failed.fetch_add(1); + } + } + finish_barrier.arrive_and_wait(); + engine.unregisterLocalMemory(source.data()); + engine.unregisterLocalMemory(destination.data()); + }); + } + start_barrier.arrive_and_wait(); + const auto started = Clock::now(); + finish_barrier.arrive_and_wait(); + const auto finished = Clock::now(); + for (auto& worker : workers) worker.join(); + if (setup_failed.load()) { + LOG(ERROR) << "failed to register benchmark buffers"; + return 2; + } + const auto seconds = + std::chrono::duration(finished - started).count(); + latencies.resize(static_cast(next.load() > FLAGS_operations + ? FLAGS_operations + : next.load())); + const double gib = static_cast(succeeded.load()) * + static_cast(FLAGS_object_size) * 2.0 / + (1024.0 * 1024.0 * 1024.0); + std::cout << "{\n" + << " \"production_equivalent_data_plane\": true,\n" + << " \"master_rpc\": \"" << FLAGS_master << "\",\n" + << " \"submaster_routing\": " + << (!FLAGS_routing_etcd.empty() ? "true" : "false") << ",\n" + << " \"cluster_namespace\": \"" << FLAGS_cluster_namespace + << "\",\n" + << " \"transfer_protocol\": \"" << FLAGS_protocol << "\",\n" + << " \"auto_discovery\": " + << (FLAGS_auto_discovery ? "true" : "false") << ",\n" + << " \"object_size_bytes\": " << FLAGS_object_size << ",\n" + << " \"concurrency\": " << FLAGS_concurrency << ",\n" + << " \"operations\": " << FLAGS_operations << ",\n" + << " \"succeeded\": " << succeeded.load() << ",\n" + << " \"failed\": " << failed.load() << ",\n" + << " \"duration_seconds\": " << seconds << ",\n" + << " \"transactions_per_second\": " + << succeeded.load() / seconds << ",\n" + << " \"data_gib_per_second\": " << gib / seconds << ",\n" + << " \"latency_p50_us\": " << Percentile(latencies, 0.50) + << ",\n" + << " \"latency_p99_us\": " << Percentile(latencies, 0.99) + << "\n}\n"; + return failed.load() == 0 && succeeded.load() == FLAGS_operations ? 0 : 1; +} diff --git a/mooncake-store/conf/master.json b/mooncake-store/conf/master.json index f55bdaf2b2..01757fdc2d 100644 --- a/mooncake-store/conf/master.json +++ b/mooncake-store/conf/master.json @@ -15,6 +15,14 @@ "enable_ha": false, "enable_oplog": false, "etcd_endpoints": "http://localhost:2379", + "enable_vchunk": false, + "vchunk_etcd_endpoints": "", + "vchunk_creating_timeout_ms": 30000, + "vchunk_releasing_timeout_ms": 60000, + "vchunk_max_slice_retry": 3, + "vchunk_max_slice_count": 4096, + "vchunk_max_metadata_bytes": 1048576, + "vchunk_max_creating_objects": 1024, "root_fs_dir": "", "cluster_id": "mooncake_cluster", "memory_allocator": "offset", diff --git a/mooncake-store/conf/master.yaml b/mooncake-store/conf/master.yaml index 545807008f..7f9bfd7f94 100644 --- a/mooncake-store/conf/master.yaml +++ b/mooncake-store/conf/master.yaml @@ -21,6 +21,16 @@ tenant_quota_connector_uri: "" enable_ha: false etcd_endpoints: "http://localhost:2379" +# Distributed vchunk validation path. Keep disabled for the community path. +enable_vchunk: false +# Empty means reuse etcd_endpoints. +vchunk_etcd_endpoints: "" +vchunk_creating_timeout_ms: 30000 +vchunk_releasing_timeout_ms: 60000 +vchunk_max_slice_retry: 3 +vchunk_max_slice_count: 4096 +vchunk_max_metadata_bytes: 1048576 +vchunk_max_creating_objects: 1024 root_fs_dir: "" cluster_id: "mooncake_cluster" memory_allocator: "offset" diff --git a/mooncake-store/include/client_service.h b/mooncake-store/include/client_service.h index f4ab18fccc..108996792a 100644 --- a/mooncake-store/include/client_service.h +++ b/mooncake-store/include/client_service.h @@ -454,6 +454,18 @@ class Client { virtual tl::expected PromotionObjectHeartbeat( std::vector& promotion_objects); + /** + * @brief Drain the removed_keys queue from master. Returns {tenant_id, + * key} pairs that were removed via Remove/BatchRemove and had LOCAL_DISK + * replicas on this client. The caller should MarkRemoved each key to + * trigger SSD tombstone + GC compaction. + */ + [[nodiscard]] tl::expected, ErrorCode> + RemoveObjectHeartbeat(const UUID& client_id); + + tl::expected AckRemoveObjectHeartbeat( + const UUID& client_id, const std::vector& tasks); + /** * @brief Stage a PROCESSING MEMORY replica for an existing key during * L2->L1 promotion. Returns the new replica's descriptor that the caller @@ -896,8 +908,30 @@ class Client { std::mutex leader_switch_mutex_; std::optional current_master_view_; std::string direct_master_address_; + // Etcd endpoints parsed from the HA backend entry, used to load the KV + // partition routing snapshot. Empty in non-HA mode. + std::string etcd_connstring_; + // Resolved cluster namespace (cluster_id) cached after the first routing + // load; reused by the periodic routing refresh loop. + std::string cluster_id_; + // CVM multi-submaster mode: set when the client bootstraps from the + // /cvm//masters/ registry (no single-leader master_view in etcd). + // Empty in single-leader HA mode and direct mode. Also drives the + // ping-failure re-discovery path. + std::string cvm_cluster_namespace_; + // CVM 多 submaster：已发现的全部 primary submaster 地址（含当前连接的 + // 那个）。用于 client 侧全量 mount（同一 segment 挂到所有 submaster）与 + // 多 submaster 心跳（防止各 submaster 因收不到 ping 而误卸载 segment）。 + // 由 ConnectToCvmSubmasters 填充。 + std::mutex cvm_submaster_addresses_mutex_; + std::vector cvm_submaster_addresses_; + // 串行化 TryLoadRoutingOnce（及其内部对 cluster_id_ 的读写）， + // 用于 routing-refresh 线程与 SLOT_NOT_OWNED 触发的按需刷新之间。 + std::mutex routing_load_mutex_; std::thread leader_monitor_thread_; std::atomic leader_monitor_running_{false}; + std::thread routing_refresh_thread_; + std::atomic routing_refresh_running_{false}; std::thread storage_heartbeat_thread_; std::atomic storage_heartbeat_running_{false}; std::thread task_poll_thread_; @@ -906,8 +940,25 @@ class Client { std::atomic segment_desc_publish_pending_{false}; std::atomic rpc_meta_publish_pending_{false}; ErrorCode SwitchLeader(const ha::MasterView& target_view); + // CVM multi-submaster bootstrap/failover: connect the etcd store client, + // discover the cluster namespace from /cvm//masters/, and connect to + // the first live primary submaster (first-registered first, matching the + // CvmController ranking). On success sets cvm_cluster_namespace_. + ErrorCode ConnectToCvmSubmasters(const std::string& etcd_endpoints); + void LoadPartitionRouting(); + void TryLoadRoutingOnce(); + void RoutingRefreshThreadMain(); void LeaderMonitorThreadMain(); void StorageHeartbeatThreadMain(); + // CVM 多 submaster：向所有 primary submaster 定向 Ping（不切换当前连接）， + // 各 submaster 收到 ping 后会保活该 client；返回 NEED_REMOUNT 的 submaster + // 会被重新 mount（重新注册 segment）。单 submaster 失败不影响其他。 + void HeartbeatAllSubmasters(); + // CVM 多 submaster：重新扫描 /cvm//masters/ 注册表，更新已发现的 + // primary 地址列表；对新增的 submaster 全量 mount 所有本地 segment，使 + // 新加入的 submaster 也具备给本 client 分配副本的能力。由路由刷新线程 + // 周期调用。 + void RefreshSubmasterAddresses(); void TaskPollThreadMain(); void EnsureStorageControlPlaneStarted(); void PollAndDispatchTasks(); diff --git a/mooncake-store/include/cvm/cvm_controller.h b/mooncake-store/include/cvm/cvm_controller.h new file mode 100644 index 0000000000..b581cc0980 --- /dev/null +++ b/mooncake-store/include/cvm/cvm_controller.h @@ -0,0 +1,163 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cvm/cvm_types.h" +#include "mutex.h" +#include "types.h" + +namespace mooncake { +namespace cvm { + +class CvmServiceDelegate; +class CvmHttpServer; + +// In-process control plane for the CVM (Cache View Master). +// +// Responsibilities (P1 skeleton): +// - Register this master under an etcd lease (liveness + role). +// - Load and continuously watch the KV view (slot -> primary master). +// - Expose local read access to the cached view (OwnsSlot / GetKvView). +// +// The full slot-migration / failover / segment-view state machines are added +// in later phases. +class CvmController { + public: + struct Config { + std::string cluster_namespace; + std::string master_id; + std::string address; // RPC endpoint of this master. + MasterRole role = MasterRole::kPrimary; + int64_t registration_lease_ttl_sec = 3; + + // CvmHttpServer（外部 HTTP 接口）配置；http_port == 0 表示不启动。 + std::string http_host = "0.0.0.0"; + uint16_t http_port = 0; + + // 集群中允许同时 serving 的 submaster 上限（名额协调，先到先得）。 + // 排名前 submaster_count 个 master 为 kPrimary，其余降级为 kStandby。 + uint32_t submaster_count = 1; + + // 视图快照生成（SyncOnce）的调度周期。 + std::chrono::milliseconds sync_interval{5000}; + }; + + explicit CvmController(Config config); + ~CvmController(); + + CvmController(const CvmController&) = delete; + CvmController& operator=(const CvmController&) = delete; + + void SetDelegate(CvmServiceDelegate* delegate); + + // 当前角色（随名额协调动态变化）。 + MasterRole GetCurrentRole() const { return current_role_.load(); } + + // 排名第一（先到先得）的 primary 的 RPC 地址，作为 standby 的单源回放 + // 目标。当成员列表为空或本机即为排名第一的 primary 时返回空字符串。 + std::string GetPrimaryAddress(); + + // 排名前 submaster_count 的 primary（submaster）成员（含 master_id 与 + // address），作为 standby 的多源回放目标。排除本机；成员列表为空或本机 + // 覆盖全部 primary 名额时返回空列表。 + std::vector GetPrimaryPeers(); + + // standby 动态绑定（2c）：按「本 standby 负责的 slot 区间」过滤出拥有这些 + // slot 的 primary，作为回放源（而非回放全部 primary）。本机为 primary 或 + // 无法确定负责区间时返回空列表。 + std::vector GetBindingSources(); + + ErrorCode Start(); + void Stop(); + + // 调度 EtcdViewStore 聚合原始记录 -> 生成视图快照 -> 回写 etcd。 + ErrorCode SyncOnce(); + + // 把最新视图快照路径推送给 CvmHttpServer。 + void PushViewPaths(); + + // etcd lease id backing this master's registration. Callers may reuse it + // for their own records (slot/segment ownership) so they share the same + // lifecycle and are auto-removed on master death. 0 until Start() succeeds. + EtcdLeaseId GetLeaseId() const { return lease_id_; } + + // Whether this master currently owns `slot` as primary. + bool OwnsSlot(uint16_t slot) const; + + // Snapshot of the cached KV view and its version. + std::vector GetKvView() const; + ViewVersionId GetKvViewVersion() const; + + private: + struct WatchState { + std::mutex mutex; + std::condition_variable cv; + bool dirty = false; + bool broken = false; + }; + + Config config_; + CvmServiceDelegate* delegate_ = nullptr; + + mutable SharedMutex view_mutex_; + std::unordered_map kv_view_; + ViewVersionId kv_view_version_{0}; + + EtcdLeaseId lease_id_{0}; + std::atomic running_{false}; + std::atomic watch_armed_{false}; + std::atomic masters_watch_armed_{false}; + + // 当前角色（随名额协调动态变化）；初始为启动配置的 role。 + std::atomic current_role_{MasterRole::kPrimary}; + + std::unique_ptr watch_state_; + std::unique_ptr masters_watch_state_; + std::thread watch_thread_; + std::thread masters_watch_thread_; + std::thread keepalive_thread_; + std::thread sync_thread_; + std::thread membership_thread_; + + std::mutex sync_mutex_; + std::condition_variable sync_cv_; + + std::mutex membership_mutex_; + std::condition_variable membership_cv_; + + // 视图类型 -> etcd 快照路径（"kv_view"/"segment_view"）。 + std::unordered_map view_paths_; + std::unique_ptr http_server_; + + ErrorCode RefreshKvView(); + void CancelWatchAndWait(); + void WatchLoop(); + void MastersWatchLoop(); + void KeepaliveLoop(); + void SyncLoop(); + void MembershipLoop(); + // 重算并回写角色（先到先得排名）；membership 轮询与 masters watch 回调 + // 共用，用 CAS 去重，保证并发下仅一次迁移与通知。 + void ReconcileRole(); + MasterRole ComputeDesiredRole(); + // 加载所有存活 master 并按「先到先得」排序（registered_at_ms 升序， + // tie-break master_id 字典序）。失败返回 false。 + bool LoadRankedMembers(std::vector& out); + + static void WatchCallback(void* ctx, const char* key, size_t key_size, + const char* value, size_t value_size, + int event_type, int64_t mod_revision); +}; + +} // namespace cvm +} // namespace mooncake diff --git a/mooncake-store/include/cvm/cvm_http_server.h b/mooncake-store/include/cvm/cvm_http_server.h new file mode 100644 index 0000000000..13b22511ca --- /dev/null +++ b/mooncake-store/include/cvm/cvm_http_server.h @@ -0,0 +1,61 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include "types.h" + +namespace mooncake { +namespace cvm { + +// CVM 三模块之一：外部 HTTP 接口（cvmhttpserver）。 +// +// 提供只读 HTTP API，按 etcd 中的视图快照路径读取快照并返回 JSON，供网页 / +// 外部客户端展示。视图快照路径由 CvmController 定期推送；未推送时使用基于 +// cluster_namespace 构造的默认快照路径（见 cvm_keys.h）。 +class CvmHttpServer { + public: + struct Config { + std::string host = "0.0.0.0"; + uint16_t port = 0; + std::string cluster_namespace; + }; + + explicit CvmHttpServer(Config config); + ~CvmHttpServer(); + + CvmHttpServer(const CvmHttpServer&) = delete; + CvmHttpServer& operator=(const CvmHttpServer&) = delete; + + ErrorCode Start(); + void Stop(); + + // 供 CvmController 推送最新视图快照路径（etcd key）。 + void SetKvViewSnapshotKey(const std::string& key); + void SetSegmentViewSnapshotKey(const std::string& key); + + // 按当前路径读取快照并返回 JSON 字符串；路径为空或读取失败时返回空串。 + std::string GetKvViewJson() const; + std::string GetSegmentViewJson() const; + + private: + void InitRoutes(); + std::string ReadSnapshot(const std::string& key) const; + + Config config_; + std::unique_ptr server_; + + mutable std::mutex path_mutex_; + std::string kv_view_snapshot_key_; + std::string segment_view_snapshot_key_; + + std::atomic running_{false}; +}; + +} // namespace cvm +} // namespace mooncake diff --git a/mooncake-store/include/cvm/cvm_keys.h b/mooncake-store/include/cvm/cvm_keys.h new file mode 100644 index 0000000000..0e1163ecc9 --- /dev/null +++ b/mooncake-store/include/cvm/cvm_keys.h @@ -0,0 +1,100 @@ +#pragma once + +#include +#include +#include +#include + +namespace mooncake { +namespace cvm { + +// Root prefix for all CVM keys in etcd. +inline constexpr std::string_view kCvmRootPrefix = "/cvm/"; + +// End key for an etcd range scan over [prefix, PrefixEnd(prefix)). +inline std::string PrefixEnd(std::string prefix) { + for (int i = static_cast(prefix.size()) - 1; i >= 0; --i) { + unsigned char c = static_cast(prefix[i]); + if (c < 0xFF) { + prefix[i] = static_cast(c + 1); + prefix.resize(i + 1); + return prefix; + } + } + return std::string(1, '\0'); +} + +// "/cvm//" +inline std::string CvmNamespaceRoot(const std::string& cluster_namespace) { + return std::string(kCvmRootPrefix) + cluster_namespace + "/"; +} + +// "/cvm//kv_view/" +inline std::string KvViewPrefix(const std::string& cluster_namespace) { + return CvmNamespaceRoot(cluster_namespace) + "kv_view/"; +} + +// "/cvm//kv_view/slot/" +inline std::string SlotOwnerKey(const std::string& cluster_namespace, + uint16_t slot) { + char buf[16]; + std::snprintf(buf, sizeof(buf), "%05u", static_cast(slot)); + return KvViewPrefix(cluster_namespace) + "slot/" + buf; +} + +// "/cvm//slot_meta/" +inline std::string SlotMetadataExportPrefix( + const std::string& cluster_namespace) { + return CvmNamespaceRoot(cluster_namespace) + "slot_meta/"; +} + +// "/cvm//slot_meta/" +// Binary (struct_pack) value holding the object metadata exported by the +// previous live primary owner during a slot handoff. +inline std::string SlotMetadataExportKey(const std::string& cluster_namespace, + uint16_t slot) { + char buf[16]; + std::snprintf(buf, sizeof(buf), "%05u", static_cast(slot)); + return SlotMetadataExportPrefix(cluster_namespace) + buf; +} + +// "/cvm//segment_view/" +inline std::string SegmentViewPrefix(const std::string& cluster_namespace) { + return CvmNamespaceRoot(cluster_namespace) + "segment_view/"; +} + +// "/cvm//segment_view/" +inline std::string SegmentOwnerKey(const std::string& cluster_namespace, + const std::string& segment_id) { + return SegmentViewPrefix(cluster_namespace) + segment_id; +} + +// "/cvm//masters/" +inline std::string MasterRegistrationPrefix( + const std::string& cluster_namespace) { + return CvmNamespaceRoot(cluster_namespace) + "masters/"; +} + +// "/cvm//masters/" +inline std::string MasterRegistrationKey(const std::string& cluster_namespace, + const std::string& master_id) { + return MasterRegistrationPrefix(cluster_namespace) + master_id; +} + +// "/cvm//snapshot/" +inline std::string SnapshotPrefix(const std::string& cluster_namespace) { + return CvmNamespaceRoot(cluster_namespace) + "snapshot/"; +} + +// "/cvm//snapshot/kv_view" +inline std::string KvViewSnapshotKey(const std::string& cluster_namespace) { + return SnapshotPrefix(cluster_namespace) + "kv_view"; +} + +// "/cvm//snapshot/segment_view" +inline std::string SegmentViewSnapshotKey(const std::string& cluster_namespace) { + return SnapshotPrefix(cluster_namespace) + "segment_view"; +} + +} // namespace cvm +} // namespace mooncake diff --git a/mooncake-store/include/cvm/cvm_service_delegate.h b/mooncake-store/include/cvm/cvm_service_delegate.h new file mode 100644 index 0000000000..81c4c65208 --- /dev/null +++ b/mooncake-store/include/cvm/cvm_service_delegate.h @@ -0,0 +1,36 @@ +#pragma once + +#include + +#include "cvm/cvm_types.h" + +namespace mooncake { +namespace cvm { + +// Callback interface implemented by MasterService. CvmController calls into +// this delegate to notify the embedding master of slot ownership changes, +// without taking a dependency on the concrete MasterService type. +class CvmServiceDelegate { + public: + virtual ~CvmServiceDelegate() = default; + + // Called when this master becomes the stable primary owner of `slot`. + virtual void OnSlotAcquired(uint16_t slot) = 0; + + // Called when this master releases ownership of `slot`. + virtual void OnSlotReleased(uint16_t slot) = 0; + + // Called when the CVM membership coordinator decides this master's role + // should change (e.g. demoted to standby because the submaster quota is + // full, or promoted back to primary). MasterService reacts by switching + // its serving/standby state machine accordingly. + virtual void OnRoleChanged(MasterRole new_role) = 0; + + // Called when the cached slot->primary view changes (slot ownership + // rebalanced or a primary's lease expired). A standby uses this to re-bind + // its replay sources even when its own role stays kStandby. + virtual void OnKvViewChanged() {} +}; + +} // namespace cvm +} // namespace mooncake diff --git a/mooncake-store/include/cvm/cvm_types.h b/mooncake-store/include/cvm/cvm_types.h new file mode 100644 index 0000000000..ea5d71b084 --- /dev/null +++ b/mooncake-store/include/cvm/cvm_types.h @@ -0,0 +1,108 @@ +#pragma once + +#include +#include +#include + +#include "types.h" + +namespace mooncake { +namespace cvm { + +// --------------------------------------------------------------------------- +// Enumerations +// --------------------------------------------------------------------------- + +// Ownership state of a logical KV slot. +enum class SlotState : int32_t { + kStable = 0, // Served by primary_master_id. + kMigrating = 1, // Handing off to migrating_to_master_id. +}; + +// Role of a master within the CVM topology. +enum class MasterRole : int32_t { + kPrimary = 0, + kStandby = 1, +}; + +// Ownership state of a segment (segment view is reserved). +enum class SegmentOwnerState : int32_t { + kStable = 0, + kTransitioning = 1, +}; + +// --------------------------------------------------------------------------- +// KV view: slot -> primary master ownership +// --------------------------------------------------------------------------- + +// NOTE: enum fields are stored as int32_t so the records serialize/deserialize +// identically across languages and compiler settings. +struct SlotOwner { + uint16_t slot{0}; + std::string primary_master_id; + int32_t state{0}; // SlotState + std::string migrating_to_master_id; // empty when stable +}; +YLT_REFL(SlotOwner, slot, primary_master_id, state, migrating_to_master_id); + +// --------------------------------------------------------------------------- +// Segment view: segment -> owner master ownership (reserved) +// --------------------------------------------------------------------------- + +struct SegmentOwner { + std::string segment_id; + std::string owner_master_id; + int32_t state{0}; // SegmentOwnerState +}; +YLT_REFL(SegmentOwner, segment_id, owner_master_id, state); + +// --------------------------------------------------------------------------- +// Master registration: liveness + role, persisted under an etcd lease +// --------------------------------------------------------------------------- + +struct MasterRegistration { + std::string master_id; + std::string address; // RPC endpoint, e.g. "host:port" + int32_t role{0}; // MasterRole + int64_t registered_at_ms{0}; +}; +YLT_REFL(MasterRegistration, master_id, address, role, registered_at_ms); + +// --------------------------------------------------------------------------- +// Aggregated views (in-memory snapshots) +// --------------------------------------------------------------------------- + +struct KvView { + std::vector slot_owners; +}; +YLT_REFL(KvView, slot_owners); + +struct SegmentView { + std::vector segment_owners; +}; +YLT_REFL(SegmentView, segment_owners); + +// --------------------------------------------------------------------------- +// Derived snapshots (persisted back to etcd) +// --------------------------------------------------------------------------- + +// Aggregated, point-in-time view of slot ownership. EtcdViewStore builds this +// from the raw slot records and writes it back to etcd so that clients can read +// the whole mapping with a single range get instead of one key per slot. +struct KvViewSnapshot { + ViewVersionId version{0}; // Etcd revision of the raw records used. + int64_t generated_at_ms{0}; // Build time (ms since epoch). + std::vector slot_owners; +}; +YLT_REFL(KvViewSnapshot, version, generated_at_ms, slot_owners); + +// Aggregated, point-in-time view of segment ownership (reserved). +struct SegmentViewSnapshot { + ViewVersionId version{0}; // Etcd revision of the raw records used. + int64_t generated_at_ms{0}; // Build time (ms since epoch). + std::vector segment_owners; +}; +YLT_REFL(SegmentViewSnapshot, version, generated_at_ms, segment_owners); + +} // namespace cvm +} // namespace mooncake diff --git a/mooncake-store/include/cvm/etcd_view_store.h b/mooncake-store/include/cvm/etcd_view_store.h new file mode 100644 index 0000000000..8c146c9418 --- /dev/null +++ b/mooncake-store/include/cvm/etcd_view_store.h @@ -0,0 +1,134 @@ +#pragma once + +#include +#include +#include + +#include "cvm/cvm_types.h" +#include "types.h" + +namespace mooncake { +namespace cvm { + +// Serialization + etcd persistence for CVM views, built on EtcdHelper. +// +// All KV view / segment view / master registration records are stored in etcd +// under the CVM key space (see cvm_keys.h). ViewVersionId is the etcd revision +// returned by the read, so consumers can watch from it without gaps. +class EtcdViewStore { + public: + // ---- JSON serialization ---- + static ErrorCode SerializeSlotOwner(const SlotOwner& owner, + std::string& out); + static ErrorCode DeserializeSlotOwner(const std::string& in, + SlotOwner& out); + + static ErrorCode SerializeSegmentOwner(const SegmentOwner& owner, + std::string& out); + static ErrorCode DeserializeSegmentOwner(const std::string& in, + SegmentOwner& out); + + static ErrorCode SerializeMasterRegistration(const MasterRegistration& reg, + std::string& out); + static ErrorCode DeserializeMasterRegistration(const std::string& in, + MasterRegistration& out); + + static ErrorCode SerializeKvViewSnapshot(const KvViewSnapshot& snapshot, + std::string& out); + static ErrorCode DeserializeKvViewSnapshot(const std::string& in, + KvViewSnapshot& out); + static ErrorCode SerializeSegmentViewSnapshot( + const SegmentViewSnapshot& snapshot, std::string& out); + static ErrorCode DeserializeSegmentViewSnapshot(const std::string& in, + SegmentViewSnapshot& out); + + // ---- KV view ---- + static ErrorCode LoadSlotOwner(const std::string& cluster_namespace, + uint16_t slot, SlotOwner& out, + ViewVersionId& version); + static ErrorCode SaveSlotOwner(const std::string& cluster_namespace, + const SlotOwner& owner); + static ErrorCode SaveSlotOwnerWithLease(const std::string& cluster_namespace, + const SlotOwner& owner, + EtcdLeaseId lease_id); + // Batch variant: writes many slot owners in chunks (each chunk a single + // etcd txn, all bound to lease_id). Speeds up bulk acquisition (e.g. cold + // start / node add) that would otherwise issue one RPC per slot. Falls back + // to per-slot puts within a chunk if the txn fails; returns an error only + // if a final per-slot put fails. + static ErrorCode SaveSlotOwnersWithLease( + const std::string& cluster_namespace, + const std::vector& owners, EtcdLeaseId lease_id); + static ErrorCode DeleteSlotOwner(const std::string& cluster_namespace, + uint16_t slot); + static ErrorCode DeleteSlotOwnerIfOwnedBy( + const std::string& cluster_namespace, uint16_t slot, + const std::string& master_id); + static ErrorCode LoadAllSlotOwners(const std::string& cluster_namespace, + std::vector& out, + ViewVersionId& version); + + // ---- Segment view (reserved) ---- + static ErrorCode LoadSegmentOwner(const std::string& cluster_namespace, + const std::string& segment_id, + SegmentOwner& out, + ViewVersionId& version); + static ErrorCode SaveSegmentOwner(const std::string& cluster_namespace, + const SegmentOwner& owner); + static ErrorCode SaveSegmentOwnerWithLease( + const std::string& cluster_namespace, const SegmentOwner& owner, + EtcdLeaseId lease_id); + static ErrorCode DeleteSegmentOwner(const std::string& cluster_namespace, + const std::string& segment_id); + static ErrorCode LoadAllSegmentOwners(const std::string& cluster_namespace, + std::vector& out, + ViewVersionId& version); + + // ---- Master registration ---- + static ErrorCode RegisterMaster(const std::string& cluster_namespace, + const MasterRegistration& reg, + EtcdLeaseId lease_id); + static ErrorCode UpdateMasterRole(const std::string& cluster_namespace, + const std::string& master_id, + MasterRole role, EtcdLeaseId lease_id); + static ErrorCode LoadAllMasters(const std::string& cluster_namespace, + std::vector& out, + ViewVersionId& version); + + // ---- Snapshots ---- + // Reads the raw slot/segment records, aggregates them into a point-in-time + // snapshot and writes it back to etcd. `version` is set to the etcd + // revision of the raw records that were aggregated. + static ErrorCode BuildAndSaveKvViewSnapshot( + const std::string& cluster_namespace, ViewVersionId& version); + static ErrorCode BuildAndSaveSegmentViewSnapshot( + const std::string& cluster_namespace, ViewVersionId& version); + + static ErrorCode SaveKvViewSnapshot(const std::string& cluster_namespace, + const KvViewSnapshot& snapshot); + static ErrorCode SaveSegmentViewSnapshot( + const std::string& cluster_namespace, const SegmentViewSnapshot& snapshot); + + // ---- Watch ---- + using WatchCallback = void (*)(void*, const char*, size_t, const char*, + size_t, int, int64_t); + static ErrorCode WatchKvView(const std::string& cluster_namespace, + ViewVersionId start_revision, void* ctx, + WatchCallback cb); + static ErrorCode CancelWatchKvView(const std::string& cluster_namespace); + static ErrorCode WaitWatchKvViewStopped(const std::string& cluster_namespace, + int timeout_ms); + + // ---- Master membership watch ---- + // Watches the master registration prefix so member add/remove (e.g. lease + // expiry) can drive immediate role re-evaluation (P3 failover). + static ErrorCode WatchMasters(const std::string& cluster_namespace, + ViewVersionId start_revision, void* ctx, + WatchCallback cb); + static ErrorCode CancelWatchMasters(const std::string& cluster_namespace); + static ErrorCode WaitWatchMastersStopped( + const std::string& cluster_namespace, int timeout_ms); +}; + +} // namespace cvm +} // namespace mooncake diff --git a/mooncake-store/include/cvm/inter_master_rpc.h b/mooncake-store/include/cvm/inter_master_rpc.h new file mode 100644 index 0000000000..baba1a5cdb --- /dev/null +++ b/mooncake-store/include/cvm/inter_master_rpc.h @@ -0,0 +1,189 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "cvm/cvm_types.h" +#include "rpc_client_io_context.h" +#include "rpc_types.h" +#include "types.h" + +namespace mooncake { + +class WrappedMasterService; + +namespace cvm { + +// Inter-master RPC client for the CVM multi-submaster topology (plan B: +// forward allocation requests to the segment-owning submaster). +// +// Discovery is fully etcd-driven (loose coupling): a background thread +// periodically reloads the master registration table +// (/cvm//masters/) and maintains a master_id -> RPC address +// mapping. One coro_rpc client pool is cached per target address, so +// repeated forwarding reuses established connections. +// +// This first phase provides the handshake RPC used to verify the channel; +// forwarding methods (e.g. PutStart allocation forwarding to the segment +// owner) build on the same invoke_rpc core. +class InterMasterRpcClient { + public: + // Refresh period for the etcd-driven member table. Matches the CVM + // membership/sync cadence (5s) so member add/remove is picked up with + // comparable latency. + static constexpr int kRefreshIntervalMs = 5000; + + // Upper bound for the pending broadcast-free queue (eviction storms + // must not grow it unboundedly; dropped entries only leak handles until + // the segment is unmounted). + static constexpr size_t kMaxBroadcastFreeQueueSize = 100000; + + InterMasterRpcClient(); + + ~InterMasterRpcClient(); + + InterMasterRpcClient(const InterMasterRpcClient&) = delete; + InterMasterRpcClient& operator=(const InterMasterRpcClient&) = delete; + + // Starts the member-refresh loop. `cluster_namespace` selects the CVM + // namespace in etcd; empty disables the refresh thread (manual updates + // only). `self_master_id` is excluded from peer handshakes. The etcd + // client must already be connected (the supervisor's CvmController + // connects it before serve). + ErrorCode Start(const std::string& cluster_namespace, + const std::string& self_master_id = ""); + + void Stop(); + + // Replaces the member table (master_id -> address). Safe to call from + // any thread; drops addresses of members that disappeared. + void UpdateMembers(const std::vector& members); + + // Current member table snapshot. + std::vector GetMembers() const; + + // Resolves the RPC address of `master_id` from the cached member table. + std::optional ResolveAddress( + const std::string& master_id) const; + + // Handshakes the target submaster: returns its identity + ownership + // summary. Used both for channel verification and liveness probing. + tl::expected Handshake( + const std::string& master_id); + + // Handshakes every known member except `self_master_id` (logging-only + // helper for startup verification). Returns the number of successes. + size_t HandshakeAll(const std::string& self_master_id); + + // ----- Allocation forwarding (CVM plan B phase 2) ----- + + // Asks the target submaster (segment owner) to allocate memory replicas + // in its locally mounted segments. When `preferred_segments` is + // non-empty the allocation is strict: only those segments are used. + // The peer keeps the real handles alive; the caller only materializes + // dummy-allocator replicas from the returned descriptors. + tl::expected, ErrorCode> + AllocateReplicas(const std::string& master_id, const std::string& tenant_id, + const std::string& key, uint64_t slice_length, + uint64_t replica_num, + const std::vector& preferred_segments); + + // Frees replicas previously allocated via AllocateReplicas on the + // target submaster. Returns true when a keepalive entry was found. + tl::expected FreeReplicas(const std::string& master_id, + const std::string& tenant_id, + const std::string& key); + + // Fire-and-forget: broadcast the free to every known peer. Used when the + // slot owner erases an object whose handles live on a segment-owning + // peer (it does not track which peer, so all peers are asked; only the + // owner of the keepalive entry reacts). + void EnqueueBroadcastFree(const std::string& tenant_id, + const std::string& key); + + // ----- Read forwarding (CVM plan B phase 2) ----- + + // Forwards a single-key GetReplicaList to the slot-owning submaster. + // The target performs a local query only (no re-forward), so an + // inconsistent view terminates the forward chain at the first hop. + tl::expected GetReplicaList( + const std::string& master_id, const std::string& key, + const std::string& tenant_id); + + // Forwards a batch GetReplicaList to the slot-owning submaster. + std::vector> + BatchGetReplicaList(const std::string& master_id, + const std::vector& keys, + const std::string& tenant_id); + + // ----- Write forwarding (model B) ----- + + // Relays a FULL PutStart to the slot-owning submaster. The owner executes + // the complete local PutStart (alloc + metadata + keepalive) and returns + // the descriptors, which the caller relays to the client. The owner's + // OwnsSlot==true so it does not re-forward (chain stops at the first hop). + tl::expected, ErrorCode> PutStart( + const std::string& master_id, const UUID& client_id, + const std::string& key, const std::string& tenant_id, + uint64_t slice_length, const ReplicateConfig& config); + + // Upsert variant: relays a FULL UpsertStart to the slot owner so it + // overwrites-if-exists (preemption) locally rather than via PutStart. + tl::expected, ErrorCode> UpsertStart( + const std::string& master_id, const UUID& client_id, + const std::string& key, const std::string& tenant_id, + uint64_t slice_length, const ReplicateConfig& config); + + private: + // Generic sync RPC invocation against the pool of the target address. + // Defined in the .cpp (needs the complete WrappedMasterService type). + template + tl::expected invoke_rpc(const std::string& address, + Args&&... args); + + // Refreshes the member table from etcd and handshakes newly joined peers + // (channel verification + connection warmup). + void RefreshLoop(); + + // Drains the broadcast-free queue: for each task, asks every known peer + // to free the keepalive entry for (tenant, key) if it holds one. + void FreeLoop(); + + struct FreeTask { + std::string tenant_id; + std::string key; + int attempts{0}; + }; + + mutable std::mutex members_mutex_; + std::unordered_map members_; // id -> address + + std::string cluster_namespace_; + std::string self_master_id_; + std::atomic running_{false}; + std::thread refresh_thread_; + std::thread free_thread_; + std::mutex cv_mutex_; + std::condition_variable cv_; + + std::mutex free_queue_mutex_; + std::condition_variable free_cv_; + std::deque free_queue_; + + // Per-address cached coro_rpc client pools (see RpcClientPool). + RpcClientPool pool_accessor_; +}; + +} // namespace cvm +} // namespace mooncake diff --git a/mooncake-store/include/cvm/slot_hash.h b/mooncake-store/include/cvm/slot_hash.h new file mode 100644 index 0000000000..1e5330386d --- /dev/null +++ b/mooncake-store/include/cvm/slot_hash.h @@ -0,0 +1,163 @@ +#pragma once + +#include +#include +#include +#include + +#include "crc32c.h" +#include "tenant_id.h" + +namespace mooncake { +namespace cvm { + +// Number of logical slots, aligned with Redis Cluster (16384 == 2^14). +constexpr uint16_t kSlotCount = 16384; +constexpr uint16_t kSlotMask = kSlotCount - 1; // 16383 + +// Number of virtual nodes per master on the consistent-hash ring used for +// dynamic slot ownership. More vnodes yield a more balanced distribution at +// the cost of a larger ring to build/sort on each heartbeat. 128 keeps load +// skew low (~1%) while remaining cheap to recompute. +constexpr uint16_t kVnodeCount = 128; + +// Maps a stable hash value to a slot by taking the low 14 bits. This is +// equivalent to `hash % kSlotCount` but cheaper. +inline uint16_t SlotOf(uint32_t hash) { + return static_cast(hash & kSlotMask); +} + +// Computes the logical slot for a tenant-scoped key. +// +// - Default tenant: slot = hash(user_key). +// - Non-default tenant: slot = hash(tenant + '\0' + user_key), so that keys +// from different tenants are isolated while remaining stable across +// processes/compilers (unlike std::hash). +inline uint16_t KeySlot(const TenantId& tenant, const std::string& user_key) { + Crc32c crc; + if (!tenant.IsDefault()) { + crc.Extend(tenant.value().data(), tenant.value().size()); + constexpr char kSeparator = '\0'; + crc.Extend(&kSeparator, 1); + } + crc.Extend(user_key.data(), user_key.size()); + return SlotOf(crc.Final()); +} + +// Position of a master's virtual node on the consistent-hash ring, in +// [0, kSlotCount). Placement depends only on (master_id, vnode_index), so it +// is deterministic across processes/compilers and independent of the current +// master set. The vnode index is encoded as fixed little-endian bytes so the +// hash is stable regardless of host endianness. +inline uint16_t VNodePosition(const std::string& master_id, uint16_t vnode) { + Crc32c crc; + crc.Extend(master_id.data(), master_id.size()); + const uint8_t vnode_bytes[2] = { + static_cast(vnode & 0xFFu), + static_cast((vnode >> 8) & 0xFFu), + }; + crc.Extend(reinterpret_cast(vnode_bytes), + sizeof(vnode_bytes)); + return SlotOf(crc.Final()); +} + +// 一致性哈希环分配：给定去重后的 primary master_id 列表 ids 与本机 +// master_id，返回本机应拥有的 slot 集合。每个 primary 在环上放置 +// kVnodeCount 个虚拟节点，slot 归属「顺时针最近的虚拟节点」。ids 应为 +// 排序去重后的 primary 列表，且必须包含 master_id。 +// +// 虚拟节点位置只依赖 (master_id, vnode_index)，因此 primary 增删时仅该 +// primary 虚拟节点覆盖的 slot（约 1/n）发生迁移，其余 primary 的 slot 保持 +// 不变，避免 naive 均分导致的「全员 slot 平移」。 +inline std::vector ResolveOwnedSlotsOnRing( + const std::vector& ids, const std::string& master_id) { + const size_t n = ids.size(); + if (n <= 1) { + std::vector slots; + slots.reserve(kSlotCount); + for (uint16_t s = 0; s < kSlotCount; ++s) { + slots.push_back(s); + } + return slots; + } + + struct VNode { + uint16_t position; + size_t owner_index; // 指向 ids + }; + std::vector ring; + ring.reserve(n * kVnodeCount); + for (size_t i = 0; i < n; ++i) { + for (uint16_t v = 0; v < kVnodeCount; ++v) { + ring.push_back({VNodePosition(ids[i], v), i}); + } + } + // 稳定排序（position 相同按 owner_index）保证跨进程结果一致。 + std::sort(ring.begin(), ring.end(), [](const VNode& a, const VNode& b) { + if (a.position != b.position) { + return a.position < b.position; + } + return a.owner_index < b.owner_index; + }); + + std::vector slots; + slots.reserve(kSlotCount / n + 1); + for (uint16_t s = 0; s < kSlotCount; ++s) { + // 环上第一个 position >= s 的虚拟节点（越界则环绕到 ring[0]）。 + auto it = std::lower_bound( + ring.begin(), ring.end(), s, + [](const VNode& vn, uint16_t value) { return vn.position < value; }); + if (it == ring.end()) { + it = ring.begin(); + } + if (ids[it->owner_index] == master_id) { + slots.push_back(s); + } + } + return slots; +} + +// 一致性哈希环反查：给定去重排序后的 primary master_id 列表 ids 与单个 +// slot，返回该 slot 的 owner master_id。规则与 ResolveOwnedSlotsOnRing 一致 +// （slot 归属顺时针最近的虚拟节点）。ids 为空返回空串，n == 1 直接返回 +// 唯一成员（单主全量接管）。供读路径转发（非 owner → slot owner）使用。 +inline std::string ResolveSlotOwnerOnRing(const std::vector& ids, + uint16_t slot) { + const size_t n = ids.size(); + if (n == 0) { + return {}; + } + if (n == 1) { + return ids[0]; + } + + struct VNode { + uint16_t position; + size_t owner_index; // 指向 ids + }; + std::vector ring; + ring.reserve(n * kVnodeCount); + for (size_t i = 0; i < n; ++i) { + for (uint16_t v = 0; v < kVnodeCount; ++v) { + ring.push_back({VNodePosition(ids[i], v), i}); + } + } + // 稳定排序（position 相同按 owner_index）保证跨进程结果一致。 + std::sort(ring.begin(), ring.end(), [](const VNode& a, const VNode& b) { + if (a.position != b.position) { + return a.position < b.position; + } + return a.owner_index < b.owner_index; + }); + + auto it = std::lower_bound( + ring.begin(), ring.end(), slot, + [](const VNode& vn, uint16_t value) { return vn.position < value; }); + if (it == ring.end()) { + it = ring.begin(); + } + return ids[it->owner_index]; +} + +} // namespace cvm +} // namespace mooncake diff --git a/mooncake-store/include/cvm/slot_migrator.h b/mooncake-store/include/cvm/slot_migrator.h new file mode 100644 index 0000000000..128228c466 --- /dev/null +++ b/mooncake-store/include/cvm/slot_migrator.h @@ -0,0 +1,72 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "cvm/cvm_types.h" +#include "types.h" + +namespace mooncake { +namespace cvm { + +// Drives the slot ownership handoff state machine (P4). +// +// A submaster owning a set of logical slots moves each slot through a two-phase +// transition whenever ownership changes: +// +// 新获得 slot: kMigrating (migrating_to = self) -> on_acquire -> kStable +// 释放 slot: on_release -> DeleteSlotOwnerIfOwnedBy(self) +// 不变 slot: kStable (幂等 reaffirm) +// +// `kMigrating` is visible to clients so they can treat a handoff-in-progress +// slot as "not yet routable" instead of hitting a half-ready new owner. +// +// SlotMigrator only moves the *ownership records*; the actual object-metadata +// materialization/drop is the caller's responsibility via the on_acquire / +// on_release hooks (data bytes always stay in segments). +class SlotMigrator { + public: + struct Config { + std::string cluster_namespace; + std::string master_id; + EtcdLeaseId lease_id{0}; + }; + + using SlotCallback = std::function; + + explicit SlotMigrator(Config config); + ~SlotMigrator() = default; + + SlotMigrator(const SlotMigrator&) = delete; + SlotMigrator& operator=(const SlotMigrator&) = delete; + + // Hooks invoked on ownership change. on_acquire materializes object + // metadata for the slot (or is a no-op when the metadata already arrived + // via the standby-restore path); on_release drops it. + void SetOnAcquire(SlotCallback cb) { on_acquire_ = std::move(cb); } + void SetOnRelease(SlotCallback cb) { on_release_ = std::move(cb); } + + // Publishes slot ownership for `owned_slots`. Idempotent; safe to call from + // the heartbeat thread. Returns the last non-OK error (if any) but keeps + // going so a single failing slot does not block the rest. + ErrorCode Reconcile(const std::vector& owned_slots); + + private: + ErrorCode PublishMigrating(uint16_t slot); + ErrorCode PublishStable(uint16_t slot); + + Config config_; + SlotCallback on_acquire_; + SlotCallback on_release_; + std::vector last_owned_slots_; + // Reconcile 调用计数，用于把不变 slot 的 reaffirm 降频为低频安全网： + // slot key 附着在 lease 上（keepalive 保活即不过期），逐周期重写只是 + // 徒增 etcd MVCC revision，曾导致 backend 配额被写满。 + uint64_t reconcile_cycles_{0}; +}; + +} // namespace cvm +} // namespace mooncake diff --git a/mooncake-store/include/cvm/slot_owner_heartbeat.h b/mooncake-store/include/cvm/slot_owner_heartbeat.h new file mode 100644 index 0000000000..17978788e5 --- /dev/null +++ b/mooncake-store/include/cvm/slot_owner_heartbeat.h @@ -0,0 +1,94 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cvm/cvm_types.h" +#include "cvm/slot_migrator.h" +#include "types.h" + +namespace mooncake { +namespace cvm { + +// Publishes this submaster's slot ownership to etcd on a fixed interval. +// +// The submaster (a MasterService instance) owns a set of logical slots. It +// periodically writes one `SlotOwner` record per owned slot under +// `/cvm/{ns}/kv_view/slot/{slot:05d}` (see EtcdViewStore::SaveSlotOwner). +// EtcdViewStore later aggregates these raw records into a `KvViewSnapshot` +// that clients read to route keys to the right submaster. +// +// Slot ownership is published on a fixed interval. With a non-zero `lease_id` +// the records are lease-bound and auto-removed when the lease expires (master +// death), which lets the dynamic partition rebalance the freed slots. With +// `lease_id == 0` the mapping is a persistent fact re-affirmed idempotently +// (single-master fallback). +class SlotOwnerHeartbeat { + public: + struct Config { + std::string cluster_namespace; + std::string master_id; + std::chrono::milliseconds heartbeat_interval{5000}; + + // Slots owned by this submaster. When empty, the submaster owns every + // logical slot (single-master mode). + std::vector owned_slots; + + // Optional resolver that recomputes the owned slot set on every + // publish. When set, it overrides `owned_slots` and is invoked before + // each PublishOnce so ownership tracks cluster membership changes + // (dynamic partition). An empty result means the submaster owns no + // slot. The callback must be safe to invoke from the heartbeat thread. + std::function()> dynamic_slot_resolver; + + // Optional etcd lease id. When non-zero, each slot record is written + // with this lease so it is auto-deleted when the lease expires (master + // death), letting the dynamic partition rebalance the freed slots. + // When zero, slot ownership is a persistent fact re-affirmed by each + // heartbeat (single-master fallback). + EtcdLeaseId lease_id{0}; + + // Optional hooks fired on slot ownership change (P4). on_slot_acquired + // materializes object metadata for a newly-owned slot (no-op when it + // already arrived via the standby-restore path); on_slot_released drops + // it. Leave empty for ownership-publishing-only behavior. + std::function on_slot_acquired; + std::function on_slot_released; + }; + + explicit SlotOwnerHeartbeat(Config config); + ~SlotOwnerHeartbeat(); + + SlotOwnerHeartbeat(const SlotOwnerHeartbeat&) = delete; + SlotOwnerHeartbeat& operator=(const SlotOwnerHeartbeat&) = delete; + + ErrorCode Start(); + void Stop(); + + // Writes a `SlotOwner` record for every owned slot once. Idempotent; safe + // to call from the heartbeat thread or externally. + ErrorCode PublishOnce(); + + private: + void RunLoop(); + + Config config_; + std::vector owned_slots_; + // Owns the slot handoff state machine (kMigrating -> kStable / release). + SlotMigrator migrator_; + + std::atomic running_{false}; + std::mutex stop_mutex_; + std::condition_variable stop_cv_; + std::thread thread_; +}; + +} // namespace cvm +} // namespace mooncake diff --git a/mooncake-store/include/etcd_helper.h b/mooncake-store/include/etcd_helper.h index 225ed6f4c3..afd24c4187 100644 --- a/mooncake-store/include/etcd_helper.h +++ b/mooncake-store/include/etcd_helper.h @@ -71,6 +71,18 @@ class EtcdHelper { static ErrorCode BatchCreate(const std::vector& keys, const std::vector& values); + /* + * @brief Atomically put a batch of key-value pairs in a single txn, all + * bound to the given lease. Returns OK if all writes committed, + * ETCD_OPERATION_ERROR otherwise. Caller is responsible for fencing + * (the keys must already be absent / owned by self) before calling; + * it never checks existence, so it can also be reused to reaffirm + * existing lease-bound keys. + */ + static ErrorCode BatchPutWithLease(const std::vector& keys, + const std::vector& values, + EtcdLeaseId lease_id); + enum class TxnCompareKind { kValueEquals = 0, kKeyNotExists = 1, @@ -94,7 +106,9 @@ class EtcdHelper { * fail. */ static ErrorCode TxnCompareAndPut(const std::vector& compares, - const std::vector& puts); + const std::vector& puts, + const std::vector& + delete_keys = {}); /* * @brief Grant a lease from the etcd. diff --git a/mooncake-store/include/file_storage.h b/mooncake-store/include/file_storage.h index d76d50145b..2439a2a082 100644 --- a/mooncake-store/include/file_storage.h +++ b/mooncake-store/include/file_storage.h @@ -51,6 +51,13 @@ class FileStorage { */ bool ReleaseBuffer(uint64_t batch_id); + // Forward explicit-delete tombstone to the storage backend. + // For BucketStorageBackend: marks tombstone + enables GC. + // For other backends: no-op (default in StorageBackendInterface). + tl::expected MarkRemoved(const std::string& key); + tl::expected BatchMarkRemoved( + const std::vector& keys); + private: friend class FileStorageTest; friend class FileStoragePromotionTest; diff --git a/mooncake-store/include/ha/ha_types.h b/mooncake-store/include/ha/ha_types.h index 7c065724e8..ea934c8b52 100644 --- a/mooncake-store/include/ha/ha_types.h +++ b/mooncake-store/include/ha/ha_types.h @@ -93,6 +93,15 @@ struct MasterView { ViewVersionId view_version = 0; }; +// A single submaster replay source for a standby. master_id doubles as the +// per-source OpLog namespace (see ha/oplog/oplog_batch_types.h). +struct MasterSource { + std::string master_id; + std::string address; +}; + +using MasterSources = std::vector; + struct LeadershipSession { MasterView view; // Backend-issued opaque ownership token. Only the backend that created diff --git a/mooncake-store/include/ha/oplog/oplog_batch_standby_reader.h b/mooncake-store/include/ha/oplog/oplog_batch_standby_reader.h index 94277049c2..4945858523 100644 --- a/mooncake-store/include/ha/oplog/oplog_batch_standby_reader.h +++ b/mooncake-store/include/ha/oplog/oplog_batch_standby_reader.h @@ -31,7 +31,8 @@ struct OpLogBatchStandbyPollResult { class OpLogBatchStandbyReader { public: OpLogBatchStandbyReader(std::string cluster_id, HaKvBackend& backend, - OpLogApplier& applier); + OpLogApplier& applier, + std::string source_id = std::string()); OpLogBatchStandbyPollResult PollOnce(size_t max_batches = 1024); diff --git a/mooncake-store/include/ha/oplog/oplog_batch_storage.h b/mooncake-store/include/ha/oplog/oplog_batch_storage.h index 7ed3880133..2d13edca90 100644 --- a/mooncake-store/include/ha/oplog/oplog_batch_storage.h +++ b/mooncake-store/include/ha/oplog/oplog_batch_storage.h @@ -11,7 +11,11 @@ namespace mooncake { class OpLogBatchStorage { public: - OpLogBatchStorage(std::string cluster_id, HaKvBackend& backend); + // source_id is the stable master id of the submaster owning this OpLog + // stream. When empty, the legacy cluster-level (single-source) layout is + // used; when non-empty, keys are namespaced per source. + OpLogBatchStorage(std::string cluster_id, HaKvBackend& backend, + std::string source_id = std::string()); ErrorCode InitDurablePrefix(DurablePrefix& prefix); ErrorCode ReadDurablePrefix(DurablePrefix& prefix); @@ -26,6 +30,7 @@ class OpLogBatchStorage { ErrorCode RejectLegacyLayout() const; std::string cluster_id_; + std::string source_id_; HaKvBackend& backend_; bool cluster_id_valid_{false}; }; diff --git a/mooncake-store/include/ha/oplog/oplog_batch_types.h b/mooncake-store/include/ha/oplog/oplog_batch_types.h index 4e315383c1..36c3bba00c 100644 --- a/mooncake-store/include/ha/oplog/oplog_batch_types.h +++ b/mooncake-store/include/ha/oplog/oplog_batch_types.h @@ -41,10 +41,20 @@ bool ValidateOpLogBatchClusterId(const std::string& cluster_id, std::string* reason = nullptr); std::string FormatOpLogBatchId(uint64_t batch_id); -std::string BuildBatchRecordKey(const std::string& cluster_id, - uint64_t batch_id); -std::string BuildDurablePrefixKey(const std::string& cluster_id); -BatchRecordRange BuildBatchRecordRange(const std::string& cluster_id, - uint64_t after_batch_id); + +// source_id is the stable master id of the submaster that owns this OpLog +// stream. When empty, the legacy cluster-level (single-source) layout is used: +// /oplog//durable_prefix +// /oplog//batches/ +// When non-empty, the per-source layout is used: +// /oplog///durable_prefix +// /oplog///batches/ +std::string BuildBatchRecordKey(const std::string& cluster_id, uint64_t batch_id, + const std::string& source_id = std::string()); +std::string BuildDurablePrefixKey( + const std::string& cluster_id, const std::string& source_id = std::string()); +BatchRecordRange BuildBatchRecordRange( + const std::string& cluster_id, uint64_t after_batch_id, + const std::string& source_id = std::string()); } // namespace mooncake diff --git a/mooncake-store/include/ha/standby_controller.h b/mooncake-store/include/ha/standby_controller.h index f6b738a713..4cd9b48795 100644 --- a/mooncake-store/include/ha/standby_controller.h +++ b/mooncake-store/include/ha/standby_controller.h @@ -31,8 +31,7 @@ class StandbyController { virtual ~StandbyController() = default; - virtual ErrorCode StartStandby( - const std::optional& observed_leader) = 0; + virtual ErrorCode StartStandby(const MasterSources& sources) = 0; virtual void StopStandby() = 0; @@ -47,8 +46,7 @@ class StandbyController { virtual tl::expected PromoteStandbyAndExport() = 0; - virtual void UpdateObservedLeader( - const std::optional& observed_leader) = 0; + virtual void UpdateObservedLeader(const MasterSources& sources) = 0; virtual MasterRuntimeState GetStandbyRuntimeState() const = 0; diff --git a/mooncake-store/include/hot_standby_service.h b/mooncake-store/include/hot_standby_service.h index f89bd76c0f..f3fb95b2c2 100644 --- a/mooncake-store/include/hot_standby_service.h +++ b/mooncake-store/include/hot_standby_service.h @@ -14,6 +14,7 @@ #include #include "metadata_store.h" +#include "ha/ha_types.h" #include "ha/oplog/oplog_applier.h" #include "ha/oplog/oplog_types.h" #include "ha/snapshot/snapshot_provider.h" @@ -34,6 +35,9 @@ enum class OpLogBatchStandbyPollDisposition; struct HotStandbyConfig { std::string standby_id; std::string primary_address; + // Submaster replay sources (one per submaster). When empty, this standby + // has no OpLog source to follow. + std::vector sources; uint32_t replication_port{0}; uint32_t verification_interval_sec{30}; uint32_t max_replication_lag_entries{1000}; @@ -89,16 +93,33 @@ class HotStandbyService { /** * @brief Start standby runtime: snapshot bootstrap plus optional OpLog * following - * @param primary_address Address of the Primary Master (not used with - * OpLog backend-based sync) + * @param sources Submaster replay sources (one per submaster); each source's + * master_id namespaces its own OpLog stream * @param oplog_endpoints Comma-separated OpLog backend endpoints * @param cluster_id Cluster identifier for OpLog path * @return ErrorCode::OK on success */ - ErrorCode Start(const std::string& primary_address, + ErrorCode Start(const std::vector& sources, const std::string& oplog_endpoints, const std::string& cluster_id); + /** + * @brief Re-bind replay sources while keeping the accumulated metadata. + * + * Under dynamic binding (2c) a standby's responsible slot range changes as + * the cluster membership changes, so its replay source set must follow. If + * the standby is not yet running this is equivalent to Start(); if it is + * running and the source set is unchanged it is a no-op; otherwise it stops + * and restarts the replication loop with the new source set. The shared + * metadata_store_ is preserved across the restart, so already-replayed + * object metadata is not lost. + * + * @return ErrorCode::OK on success + */ + ErrorCode UpdateSources(const std::vector& sources, + const std::string& oplog_endpoints, + const std::string& cluster_id); + /** * @brief Stop replication and disconnect from Primary */ @@ -201,6 +222,7 @@ class HotStandbyService { ErrorCode StartOplogFollowingLocked(uint64_t baseline_seq_id); void ActivateSnapshotOnlyStandbyLocked(uint64_t baseline_seq_id); uint64_t GetLocalLastAppliedSequenceIdLocked() const; + void CollectSegmentsLocked(std::vector& out) const; ErrorCode FinalCatchUpForPromotionLocked(uint64_t current_applied_seq_id); ErrorCode FinalCatchUpBatchRecordsLocked(HaKvBackend& backend); void StopReplicationLoop(); @@ -257,10 +279,18 @@ class HotStandbyService { std::unique_ptr snapshot_provider_{ std::make_unique()}; - // OpLog replication components - std::unique_ptr oplog_applier_; + // Segment baseline loaded from snapshot bootstrap, applied to each + // per-source applier when OpLog following starts. + std::vector baseline_segments_; + + // Per-source OpLog replication components. Keyed by source master_id. All + // appliers share the single metadata_store_ (sources own disjoint slots). + struct SourceReplica { + std::unique_ptr applier; + std::unique_ptr reader; + }; + std::unordered_map sources_; std::shared_ptr batch_standby_kv_backend_; - std::unique_ptr batch_standby_reader_; std::shared_ptr catch_up_batch_kv_backend_for_testing_; diff --git a/mooncake-store/include/master_client.h b/mooncake-store/include/master_client.h index 8210b7d882..7dc9acc89b 100644 --- a/mooncake-store/include/master_client.h +++ b/mooncake-store/include/master_client.h @@ -1,7 +1,10 @@ #pragma once #include +#include #include +#include +#include #include #include #include @@ -22,6 +25,8 @@ #include "store_rpc_client_io_context.h" #include "task_manager.h" #include "metadata_store.h" +#include "partition/partition_router.h" +#include "vchunk_metadata.h" namespace mooncake { @@ -79,6 +84,8 @@ class MasterClient { std::string tenant_id = "default") : client_accessor_(GetStoreRpcClientIoContextPool(), detail::MakeMasterRpcClientPoolConfig()), + targeted_accessor_(GetStoreRpcClientIoContextPool(), + detail::MakeMasterRpcClientPoolConfig()), client_id_(client_id), tenant_id_(NormalizeTenantId(std::move(tenant_id))), metrics_(metrics) { @@ -122,6 +129,27 @@ class MasterClient { [[nodiscard]] ErrorCode Connect( const std::string& master_addr = kDefaultMasterAddress); + /** + * @brief Loads the slot -> submaster mapping from the etcd KV view + * snapshot into the partition router. Must be called after etcd is + * reachable and the cluster namespace (cluster_id) is known. + * @param etcd_endpoints Etcd endpoints, semicolon separated. + * @param cluster_namespace Cluster namespace used to locate the snapshot. + * @return ErrorCode indicating success/failure. + */ + [[nodiscard]] ErrorCode LoadRoutingFromEtcd( + const std::string& etcd_endpoints, + const std::string& cluster_namespace); + + /** + * @brief Resolves the submaster (primary_master_id) that owns the slot of + * the given key, using the currently loaded partition routing table. + * @param key Object key. + * @return submaster id on success, std::nullopt if no mapping is loaded. + */ + [[nodiscard]] std::optional ResolveSubmaster( + const std::string& key) const; + /** * @brief Checks if an object exists * @param object_key Key to query @@ -262,6 +290,25 @@ class MasterClient { [[nodiscard]] tl::expected PutRevoke( const std::string& key, ReplicaType replica_type); + [[nodiscard]] tl::expected VChunkPutStart( + const std::string& tenant_id, const std::string& key, + uint64_t total_size, int64_t now_ms); + [[nodiscard]] tl::expected VChunkPutEnd( + const std::string& tenant_id, const std::string& key, + const std::string& vchunk_id, int64_t now_ms); + [[nodiscard]] tl::expected VChunkPutRevoke( + const std::string& tenant_id, const std::string& key, + const std::string& vchunk_id); + [[nodiscard]] tl::expected GetVChunk( + const std::string& tenant_id, const std::string& key); + [[nodiscard]] tl::expected ReleaseVChunkReadLease( + const std::string& tenant_id, const std::string& key, + const std::string& lease_id); + [[nodiscard]] tl::expected RemoveVChunk( + const std::string& tenant_id, const std::string& key, int64_t now_ms); + [[nodiscard]] tl::expected + GetVChunkRuntimeInfo(); + /** * @brief Revokes a put operation for a batch of objects * @param keys Vector of object keys @@ -432,6 +479,14 @@ class MasterClient { [[nodiscard]] tl::expected QuerySegmentStatusById( const UUID& segment_id); + /** + * @brief 定向 QuerySegmentStatusById：向指定 submaster 查询 segment 状态 + * （不切换当前连接）。用于多 submaster 优雅卸载时确认所有 primary + * submaster 均已移除该 segment。 + */ + [[nodiscard]] tl::expected QuerySegmentStatusByIdTo( + const std::string& address, const UUID& segment_id); + [[nodiscard]] tl::expected GetStorageConfig(); @@ -442,6 +497,46 @@ class MasterClient { */ [[nodiscard]] tl::expected Ping(); + /** + * @brief 返回 client_accessor_ 当前连接的 submaster 地址（IP:Port）。 + * 供后台心跳去重使用——HeartbeatAllSubmasters 跳过当前 active 地址， + * 避免与主循环的 Ping() 对同一 submaster 重复 ping。 + */ + std::string GetCurrentAddress() const; + + /** + * @brief 定向 Ping：向指定 submaster 发送 Ping（不切换当前连接）。 + * 用于 client 侧多 submaster 心跳，防止各 submaster 因收不到 ping 而 + * 误判 client 过期并卸载其 segment。返回 NEED_REMOUNT 时调用方应对该 + * submaster 重新 mount。 + */ + [[nodiscard]] tl::expected PingTo( + const std::string& address); + + /** + * @brief 定向 MountSegment：向指定 submaster 注册 segment（不切换当前 + * 连接）。用于 client 侧全量 mount——同一 segment 挂载到所有 primary + * submaster，使任何 slot owner 都能本地分配副本。 + */ + [[nodiscard]] tl::expected MountSegmentTo( + const std::string& address, const Segment& segment); + + /** + * @brief 定向 UnmountSegment：向指定 submaster 注销 segment（不切换当前 + * 连接）。与全量 mount 对称，保证 segment 生命周期在所有 submaster 闭合。 + */ + [[nodiscard]] tl::expected UnmountSegmentTo( + const std::string& address, const UUID& segment_id); + + /** + * @brief 定向 GracefulUnmountSegment：向指定 submaster 发起优雅卸载（不 + * 切换当前连接）。与全量 unmount 对称，保证优雅卸载在所有 submaster 生效。 + */ + [[nodiscard]] tl::expected GracefulUnmountSegmentTo( + const std::string& address, const UUID& segment_id, + uint64_t grace_period_ms); + + /** * @brief Mounts a local disk segment into the master. * @param enable_offloading If true, enables offloading (write-to-file). @@ -493,6 +588,12 @@ class MasterClient { [[nodiscard]] tl::expected, ErrorCode> PromotionObjectHeartbeat(const UUID& client_id); + /** Fetch pending remove tasks without removing them from the queue. */ + [[nodiscard]] tl::expected, ErrorCode> + RemoveObjectHeartbeat(const UUID& client_id); + tl::expected AckRemoveObjectHeartbeat( + const UUID& client_id, const std::vector& tasks); + /** * @brief Stage a PROCESSING MEMORY replica for an existing key during * promotion. Returns the new replica's descriptor that the caller writes @@ -692,6 +793,16 @@ class MasterClient { [[nodiscard]] tl::expected invoke_rpc( Args&&... args); + /** + * @brief 定向 RPC：向指定 submaster 发请求，使用独立的 targeted_accessor_ + * 缓存 per-address pool，不切换 client_accessor_ 的"当前地址"。供后台心跳 + * /全量 mount/unmount 使用，避免与业务请求（SwitchToSubmaster + invoke_rpc） + * 的当前地址切换竞态。 + */ + template + [[nodiscard]] tl::expected invoke_rpc_to( + const std::string& address, Args&&... args); + /** * @brief Generic RPC invocation helper for batch operations * @tparam ServiceMethod Pointer to WrappedMasterService member function @@ -707,6 +818,34 @@ class MasterClient { void WarmupRpcPool(); + /** + * @brief Switches the underlying RPC pool to the submaster that owns the + * slot of the given key (single-target switching). Keeps the current + * connection when the routing table is not loaded (single-master mode). + * @return ErrorCode::OK on switch (or no-op), otherwise an error. + */ + [[nodiscard]] ErrorCode SwitchToSubmaster(const std::string& tenant_id, + const std::string& key); + + // Refreshes the previously configured CVM snapshot after a server reports + // SLOT_NOT_OWNED. The caller remains responsible for a bounded retry. + [[nodiscard]] ErrorCode RefreshSubmasterRouting(); + + /** + * @brief Switches the underlying RPC pool to the given submaster address. + * @param address Submaster address (primary_master_id). + */ + void SwitchToSubmasterByAddress(const std::string& address); + + /** + * @brief Groups key indices by their owning submaster. Keys without a + * resolved submaster are grouped under an empty string (""). + * @return Map from submaster address to original key indices. + */ + [[nodiscard]] std::map> + GroupKeysBySubmaster(const std::vector& keys, + const std::string& tenant_id); + /** * @brief Accessor for the coro_rpc_client pool. Since coro_rpc_client pool * cannot reconnect to a different address, a new coro_rpc_client pool is @@ -735,12 +874,26 @@ class MasterClient { RpcClientPool client_accessor_; + // 定向 RPC 用独立 pool 访问器：后台心跳/全量 mount/unmount 用它直发指定 + // submaster，与业务请求的 client_accessor_（SwitchToSubmaster 切"当前地址"） + // 完全隔离，避免"当前地址"竞态。invoke_rpc_to 用 GetOrCreateClientPool 的 + // 返回值发请求，不依赖该访问器的"当前地址"，因此可被多线程并发调用。 + RpcClientPool targeted_accessor_; + // The client identification. const UUID client_id_; // Tenant identity for this client instance. const TenantId tenant_id_; + // KV partition routing table (slot -> submaster). Loaded from etcd. + partition::PartitionRouter partition_router_; + mutable std::mutex routing_config_mutex_; + std::string routing_cluster_namespace_; + // Protects the target switch and the following vchunk RPC as one unit. + // RpcClientPool itself is thread-safe, but its selected target is shared. + mutable std::mutex vchunk_routed_rpc_mutex_; + // Metrics for tracking RPC operations MasterClientMetric* metrics_; std::shared_ptr> diff --git a/mooncake-store/include/master_config.h b/mooncake-store/include/master_config.h index b73d9ca61c..43ab2e3a24 100644 --- a/mooncake-store/include/master_config.h +++ b/mooncake-store/include/master_config.h @@ -3,11 +3,14 @@ #include #include #include +#include #include #include "config_helper.h" #include "types.h" +#include "vchunk_config.h" +#include "vchunk_metadata_store.h" namespace mooncake { @@ -74,6 +77,13 @@ struct MasterConfig { int metrics_report_lease_ttl_sec = DEFAULT_METRICS_REPORT_LEASE_TTL_SEC; std::string cluster_id; + // 集群中允许同时 serving 的 submaster 上限（CVM 名额协调，先到先得）。 + // 默认 1 保持单主行为；>1 时多 submaster 均分 slot，超出 k 名自动降级为 standby。 + uint32_t submaster_count = 1; + // CVM external HTTP API (CvmHttpServer) bind config. Port 0 keeps the + // HTTP server disabled. + uint16_t cvm_http_port = 0; + std::string cvm_http_host = "0.0.0.0"; std::string root_fs_dir; int64_t global_file_segment_size; std::string memory_allocator; @@ -132,6 +142,8 @@ struct MasterConfig { std::string cxl_path; size_t cxl_size; bool enable_cxl = false; + VChunkConfig vchunk_config{}; + std::string vchunk_etcd_endpoints; // Offload-on-evict: defer LOCAL_DISK offload to eviction time bool offload_on_evict = false; @@ -213,6 +225,12 @@ class MasterServiceSupervisorConfig { uint32_t batch_oplog_retry_timeout_sec = 180; std::string local_hostname = "0.0.0.0:50051"; std::string cluster_id = DEFAULT_CLUSTER_ID; + // 集群中允许同时 serving 的 submaster 上限（CVM 名额协调，先到先得）。 + uint32_t submaster_count = 1; + // CVM external HTTP API (CvmHttpServer) bind config. Port 0 keeps the + // HTTP server disabled. + uint16_t cvm_http_port = 0; + std::string cvm_http_host = "0.0.0.0"; // Metrics reporting to HA backend (etcd/redis). bool enable_metrics_report_to_backend = @@ -255,6 +273,8 @@ class MasterServiceSupervisorConfig { std::string cxl_path = DEFAULT_CXL_PATH; size_t cxl_size = DEFAULT_CXL_SIZE; bool enable_cxl = false; + VChunkConfig vchunk_config{}; + std::string vchunk_etcd_endpoints; bool offload_on_evict = false; bool offload_force_evict = false; bool strict_replica_allocation = false; @@ -350,6 +370,9 @@ class MasterServiceSupervisorConfig { batch_oplog_retry_timeout_sec = config.batch_oplog_retry_timeout_sec; local_hostname = rpc_address + ":" + std::to_string(rpc_port); cluster_id = config.cluster_id; + submaster_count = config.submaster_count; + cvm_http_port = config.cvm_http_port; + cvm_http_host = config.cvm_http_host; enable_metrics_report_to_backend = config.enable_metrics_report_to_backend; @@ -415,6 +438,10 @@ class MasterServiceSupervisorConfig { cxl_path = config.cxl_path; cxl_size = config.cxl_size; enable_cxl = config.enable_cxl; + vchunk_config = config.vchunk_config; + vchunk_etcd_endpoints = config.vchunk_etcd_endpoints.empty() + ? config.etcd_endpoints + : config.vchunk_etcd_endpoints; pod_name = config.pod_name; pod_namespace = config.pod_namespace; @@ -533,6 +560,18 @@ class WrappedMasterServiceConfig { int oplog_poll_interval_ms = 1000; uint32_t oplog_batch_max_entries = 1024; std::string cluster_id = DEFAULT_CLUSTER_ID; + // Stable identifier of this master instance, used as the SlotOwner + // primary_master_id for the KV partition view. Empty disables the + // SlotOwnerHeartbeat. In HA mode this is the local_hostname. + std::string master_id; + // CVM external HTTP API (CvmHttpServer) bind config. Port 0 keeps the + // HTTP server disabled; set a non-zero port to expose /kv_view, + // /segment_view and /health for inspection. + uint16_t cvm_http_port = 0; + std::string cvm_http_host = "0.0.0.0"; + // 集群中允许同时 serving 的 submaster 上限（CVM 名额协调，先到先得）。 + // 默认 1 保持单主行为；>1 时多 submaster 均分 slot，超出 k 名自动降级为 standby。 + uint32_t submaster_count = 1; std::string root_fs_dir = DEFAULT_ROOT_FS_DIR; int64_t global_file_segment_size = DEFAULT_GLOBAL_FILE_SEGMENT_SIZE; BufferAllocatorType memory_allocator = BufferAllocatorType::OFFSET; @@ -568,6 +607,8 @@ class WrappedMasterServiceConfig { std::string cxl_path = DEFAULT_CXL_PATH; size_t cxl_size = DEFAULT_CXL_SIZE; bool enable_cxl = false; + VChunkConfig vchunk_config{}; + std::string vchunk_etcd_endpoints; WrappedMasterServiceConfig() = default; // From MasterConfig @@ -624,6 +665,9 @@ class WrappedMasterServiceConfig { oplog_poll_interval_ms = config.oplog_poll_interval_ms; oplog_batch_max_entries = config.oplog_batch_max_entries; cluster_id = config.cluster_id; + submaster_count = config.submaster_count; + cvm_http_port = config.cvm_http_port; + cvm_http_host = config.cvm_http_host; root_fs_dir = config.root_fs_dir; global_file_segment_size = config.global_file_segment_size; enable_disk_eviction = config.enable_disk_eviction; @@ -683,6 +727,10 @@ class WrappedMasterServiceConfig { cxl_path = config.cxl_path; cxl_size = config.cxl_size; enable_cxl = config.enable_cxl; + vchunk_config = config.vchunk_config; + vchunk_etcd_endpoints = config.vchunk_etcd_endpoints.empty() + ? config.etcd_endpoints + : config.vchunk_etcd_endpoints; } // From MasterServiceSupervisorConfig, enable_ha is set to true @@ -741,6 +789,10 @@ class WrappedMasterServiceConfig { oplog_poll_interval_ms = config.oplog_poll_interval_ms; oplog_batch_max_entries = config.oplog_batch_max_entries; cluster_id = config.cluster_id; + master_id = config.local_hostname; + submaster_count = config.submaster_count; + cvm_http_port = config.cvm_http_port; + cvm_http_host = config.cvm_http_host; root_fs_dir = config.root_fs_dir; global_file_segment_size = config.global_file_segment_size; memory_allocator = config.memory_allocator; @@ -773,6 +825,8 @@ class WrappedMasterServiceConfig { cxl_path = config.cxl_path; cxl_size = config.cxl_size; enable_cxl = config.enable_cxl; + vchunk_config = config.vchunk_config; + vchunk_etcd_endpoints = config.vchunk_etcd_endpoints; } }; @@ -841,6 +895,8 @@ class MasterServiceConfigBuilder { std::string cxl_path_ = DEFAULT_CXL_PATH; size_t cxl_size_ = DEFAULT_CXL_SIZE; bool enable_cxl_ = false; + VChunkConfig vchunk_config_{}; + std::shared_ptr vchunk_metadata_store_; public: MasterServiceConfigBuilder() = default; @@ -985,6 +1041,17 @@ class MasterServiceConfigBuilder { return *this; } + MasterServiceConfigBuilder& set_vchunk_config(VChunkConfig config) { + vchunk_config_ = std::move(config); + return *this; + } + + MasterServiceConfigBuilder& set_vchunk_metadata_store( + std::shared_ptr store) { + vchunk_metadata_store_ = std::move(store); + return *this; + } + MasterServiceConfigBuilder& set_tenant_quota_connector_type( const std::string& type) { tenant_quota_connector_type_ = type; @@ -1192,6 +1259,18 @@ class MasterServiceConfig { int oplog_poll_interval_ms = 1000; uint32_t oplog_batch_max_entries = 1024; std::string cluster_id = DEFAULT_CLUSTER_ID; + // Stable identifier of this master instance, used as the SlotOwner + // primary_master_id for the KV partition view. Empty disables the + // SlotOwnerHeartbeat. In HA mode this is the local_hostname. + std::string master_id; + // CVM external HTTP API (CvmHttpServer) bind config. Port 0 keeps the + // HTTP server disabled; set a non-zero port to expose /kv_view, + // /segment_view and /health for inspection. + uint16_t cvm_http_port = 0; + std::string cvm_http_host = "0.0.0.0"; + // 集群中允许同时 serving 的 submaster 上限（CVM 名额协调，先到先得）。 + // 默认 1 保持单主行为；>1 时多 submaster 均分 slot，超出 k 名自动降级为 standby。 + uint32_t submaster_count = 1; std::string root_fs_dir = DEFAULT_ROOT_FS_DIR; int64_t global_file_segment_size = DEFAULT_GLOBAL_FILE_SEGMENT_SIZE; BufferAllocatorType memory_allocator = BufferAllocatorType::OFFSET; @@ -1227,6 +1306,9 @@ class MasterServiceConfig { std::string cxl_path = DEFAULT_CXL_PATH; size_t cxl_size = DEFAULT_CXL_SIZE; bool enable_cxl = false; + VChunkConfig vchunk_config{}; + std::string vchunk_etcd_endpoints; + std::shared_ptr vchunk_metadata_store; MasterServiceConfig() = default; // From WrappedMasterServiceConfig @@ -1277,6 +1359,10 @@ class MasterServiceConfig { oplog_poll_interval_ms = config.oplog_poll_interval_ms; oplog_batch_max_entries = config.oplog_batch_max_entries; cluster_id = config.cluster_id; + master_id = config.master_id; + cvm_http_port = config.cvm_http_port; + cvm_http_host = config.cvm_http_host; + submaster_count = config.submaster_count; root_fs_dir = config.root_fs_dir; global_file_segment_size = config.global_file_segment_size; memory_allocator = @@ -1315,6 +1401,12 @@ class MasterServiceConfig { cxl_path = config.cxl_path; cxl_size = config.cxl_size; enable_cxl = config.enable_cxl; + vchunk_config = config.vchunk_config; + vchunk_etcd_endpoints = config.vchunk_etcd_endpoints; + if (vchunk_config.enabled && !vchunk_etcd_endpoints.empty()) { + vchunk_metadata_store = std::make_shared( + vchunk_etcd_endpoints, vchunk_config, cluster_id); + } } // Static factory method to create a builder @@ -1381,6 +1473,8 @@ inline MasterServiceConfig MasterServiceConfigBuilder::build() const { config.cxl_path = cxl_path_; config.cxl_size = cxl_size_; config.enable_cxl = enable_cxl_; + config.vchunk_config = vchunk_config_; + config.vchunk_metadata_store = vchunk_metadata_store_; return config; } diff --git a/mooncake-store/include/master_service.h b/mooncake-store/include/master_service.h index 9b758ea7ff..2c17142084 100644 --- a/mooncake-store/include/master_service.h +++ b/mooncake-store/include/master_service.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -25,6 +26,10 @@ #include "allocation_strategy.h" #include "count_min_sketch.h" +#include "cvm/cvm_controller.h" +#include "cvm/inter_master_rpc.h" +#include "cvm/slot_hash.h" +#include "cvm/slot_owner_heartbeat.h" #include "deadline_scheduler.h" #include "master_metric_manager.h" #include "mutex.h" @@ -32,6 +37,7 @@ #include "tenant_quota_sharded.h" #include "tenant_quota_policy_store.h" #include "types.h" +#include "vchunk_master_manager.h" #include "master_config.h" #include "rpc_types.h" #include "replica.h" @@ -142,6 +148,80 @@ class MasterService { ErrorCode SetBatchOpLogBackendForTesting( std::shared_ptr backend); + // CVM slot ownership publishing. The CvmController now lives in the HA + // supervisor so membership keeps running even while this node is a + // standby; the supervisor injects the lease id it granted and drives the + // heartbeat around serve start/stop. + void SetCvmLeaseId(EtcdLeaseId lease_id); + ErrorCode StartSlotOwnerHeartbeat(); + void StopSlotOwnerHeartbeat(); + + // CVM inter-master RPC (multi-submaster coordination). Starts the + // etcd-driven member refresh loop so this master can reach its peers + // (handshake now; allocation forwarding later). Same lifecycle as the + // slot owner heartbeat: called by the supervisor around serve phases. + ErrorCode StartInterMasterRpc(); + void StopInterMasterRpc(); + cvm::InterMasterRpcClient* inter_master_rpc() const { + return inter_master_rpc_.get(); + } + + // Inter-master handshake data: stable identity, supervisor-granted lease + // and current owned-slot count (read path / forwarding diagnostics). + const std::string& master_id() const { return master_id_; } + EtcdLeaseId cvm_lease_id() const { return cvm_lease_id_; } + uint32_t GetOwnedSlotCount() const; + + // Inter-master allocation forwarding (CVM plan B phase 2). Called by + // WrappedMasterService when a slot-owning peer asks this submaster + // (segment owner) to allocate memory replicas. `preferred_segments` + // is honored strictly when non-empty. Real handles are kept alive in + // a keepalive registry until InterMasterFreeReplicas arrives. + tl::expected, ErrorCode> + InterMasterAllocateReplicas(const std::string& tenant_id, + const std::string& key, uint64_t slice_length, + uint64_t replica_num, + const std::vector& preferred_segments); + + // Frees the keepalive entry (and the handles) for (tenant, key). + // Idempotent: returns false when no entry exists. + tl::expected InterMasterFreeReplicas( + const std::string& tenant_id, const std::string& key); + + // Inter-master read forwarding (CVM plan B phase 2). Called by + // WrappedMasterService when a peer submaster forwards a GetReplicaList / + // BatchGetReplicaList request to this submaster (the slot owner). Unlike + // the client-facing GetReplicaList, these do NOT re-forward: they query + // the local metadata directly (peer trust), so an inconsistent view + // terminates the forward chain at the first hop instead of looping. + tl::expected InterMasterGetReplicaList( + const std::string& key, const std::string& tenant_id); + + std::vector> + InterMasterBatchGetReplicaList(const std::vector& keys, + const std::string& tenant_id); + + // Inter-master write forwarding (model B). Called by WrappedMasterService + // when a peer submaster received a PutStart for a slot it does not own: it + // relays the FULL PutStart to this submaster (the slot owner), which + // executes the complete local alloc + metadata + keepalive and returns the + // descriptors for the caller to relay to the client. Because this is the + // slot owner, its OwnsSlot==true so the forward chain terminates here + // (peer trust, symmetric with InterMasterGetReplicaList). + tl::expected, ErrorCode> InterMasterPutStart( + const UUID& client_id, const std::string& key, + const std::string& tenant_id, uint64_t slice_length, + const ReplicateConfig& config); + + // Upsert variant of InterMasterPutStart: the slot owner executes the FULL + // UpsertStart (preserving the "overwrite if exists" preemption semantics + // that PutStart lacks), and returns the descriptors for the caller to + // relay to the client. + tl::expected, ErrorCode> + InterMasterUpsertStart(const UUID& client_id, const std::string& key, + const std::string& tenant_id, uint64_t slice_length, + const ReplicateConfig& config); + /** * @brief Test-only wrapper around BatchEvict / NoFBatchEvict so that * unit tests can drive a single eviction cycle synchronously @@ -164,6 +244,29 @@ class MasterService { auto MountSegment(const Segment& segment, const UUID& client_id) -> tl::expected; + tl::expected VChunkPutStart( + const TenantId& tenant_id, const std::string& key, + uint64_t total_size, bool is_ssd_segment, int64_t now_ms, + const std::set& excluded_segments = {}); + ErrorCode VChunkPutEnd(const TenantId& tenant_id, const std::string& key, + const std::string& vchunk_id, int64_t now_ms); + ErrorCode VChunkPutRevoke(const TenantId& tenant_id, + const std::string& key, + const std::string& vchunk_id); + tl::expected GetVChunk( + const TenantId& tenant_id, const std::string& key) const; + tl::expected AcquireVChunkRead( + const TenantId& tenant_id, const std::string& key) const; + tl::expected AcquireVChunkReadLease( + const TenantId& tenant_id, const std::string& key, int64_t now_ms); + ErrorCode ReleaseVChunkReadLease(const std::string& lease_id); + ErrorCode RemoveVChunk(const TenantId& tenant_id, const std::string& key, + int64_t now_ms); + VChunkRuntimeInfo GetVChunkRuntimeInfo() const; + tl::expected ReapExpiredVChunks(int64_t now_ms, + size_t max_scan); + VChunkMetricsSnapshot GetVChunkMetrics() const; + /** * @brief Mount a NoF SSD segment for buffer allocation. This function is * idempotent. @@ -727,6 +830,13 @@ class MasterService { auto PromotionObjectHeartbeat(const UUID& client_id) -> tl::expected, ErrorCode>; + /** Fetch pending remove tasks without removing them from the queue. */ + auto RemoveObjectHeartbeat(const UUID& client_id) + -> tl::expected, ErrorCode>; + auto AckRemoveObjectHeartbeat( + const UUID& client_id, const std::vector& tasks) + -> tl::expected; + /** * @brief Stage a PROCESSING MEMORY replica for an existing key. Allocates * DRAM via the existing AllocationStrategy, optionally biased toward the @@ -1464,20 +1574,23 @@ class MasterService { bool IsTenantRegistered(const TenantId& tenant_id) const; bool TenantHasObjects(const TenantId& tenant_id) const; - // Helper to get shard index from tenant-scoped object identity. + // Helper to get the logical slot (Redis Cluster style, 16384 slots) for + // a tenant-scoped key. Slots are the unit of cross-master ownership. + uint16_t getSlot(const TenantId& tenant_id, + const std::string& user_key) const { + return cvm::KeySlot(tenant_id, user_key); + } + + // Helper to get shard index from tenant-scoped object identity. The + // physical shard is derived from the logical slot. size_t getShardIndex(const TenantId& tenant_id, const std::string& user_key) const { - if (tenant_id.IsDefault()) { - return std::hash{}(user_key) % kNumShards; - } - size_t seed = std::hash{}(tenant_id.value()); - boost::hash_combine(seed, user_key); - return seed % kNumShards; + return getSlot(tenant_id, user_key) % kNumShards; } // Legacy helper routes plain keys to the default tenant. size_t getShardIndex(const std::string& key) const { - return std::hash{}(key) % kNumShards; + return getSlot(TenantId::Default(), key) % kNumShards; } size_t getMetadataShardIndex(const TenantId& tenant_id, @@ -1500,6 +1613,9 @@ class MasterService { kFull, kPreserveOld, kAbortOnly, + // slot 交接：与 kFull 相同，但跳过 ReleaseLocalDiskUsage，因为数据字节 + // 仍留在共享 segment（不应错误扣减 ssd_used_bytes）。 + kHandoff, }; std::unordered_map::iterator EraseMetadata( TenantState& tenant_state, @@ -1533,6 +1649,8 @@ class MasterService { void FinalizeRemovedReplicasAfterDurable( const OpLogEntry& durable_entry, const std::vector& replica_ids, QuotaEraseMode quota_mode); + void EnqueueRemoveTasks(const std::vector& holder_ids, + const RemoveTaskItem& task); void FinalizeMetadataEraseAfterDurable(const OpLogEntry& durable_entry, QuotaEraseMode quota_mode); void FinalizeExpiredProcessingReplicasAfterDurable( @@ -2094,6 +2212,114 @@ class MasterService { // cluster id for persistent sub directory const std::string cluster_id_; + // Stable master id used by the SlotOwnerHeartbeat (empty = disabled). + const std::string master_id_; + // CVM external HTTP API (CvmHttpServer) bind config. Port 0 keeps the + // HTTP server disabled. + const uint16_t cvm_http_port_; + const std::string cvm_http_host_; + // 集群中允许同时 serving 的 submaster 上限（CVM 名额协调）。 + const uint32_t submaster_count_; + // Publishes this submaster's slot ownership to etcd for the KV partition + // view. Only started in HA mode when a stable master_id is configured. + std::unique_ptr slot_owner_heartbeat_; + // Inter-master RPC client (CVM multi-submaster coordination): etcd-driven + // member table + per-peer cached coro_rpc pools. Started/stopped by the + // supervisor around serve phases, mirroring the heartbeat above. + std::unique_ptr inter_master_rpc_; + + // ----- Inter-master allocation forwarding (CVM plan B phase 2) ----- + + // Segment-owner side: keepalive for replicas allocated on behalf of + // slot-owning peers, keyed by scoped key. The real handles stay alive + // here; the peer only holds dummy-allocator replicas. Freed by + // InterMasterFreeReplicas (broadcast by the slot owner on erase). + std::mutex inter_master_keepalive_mutex_; + std::unordered_map> + inter_master_keepalive_; + + // Slot-owner side: scoped keys whose handles live on a peer submaster + // (remote-allocated or migrated-in). Drives the broadcast-free on + // EraseMetadata; kHandoff (slot migration) keeps the handles alive and + // transfers the responsibility to the importing master. + std::mutex remote_allocated_keys_mutex_; + std::unordered_set remote_allocated_keys_; + + // Slot-owner side: keeps the DummyBufferAllocator of remote replicas + // alive (the replicas hold only weak references), keyed by endpoint. + std::mutex remote_replica_allocator_keepalive_mutex_; + std::unordered_map> + remote_replica_allocator_keepalive_; + + // Slot-owner side: local allocation failed, try peers (segment owners) + // before giving up. Called OUTSIDE the local allocator lock to avoid + // distributed deadlock (both masters forwarding simultaneously). + tl::expected, ErrorCode> TryAllocateReplicasRemotely( + const std::string& key, const TenantId& tenant_id, + uint64_t value_length, size_t replica_num, + const std::vector& preferred_segments); + + // EraseMetadata hook: when the erased object's handles live on a peer, + // enqueue a broadcast free (skipped for kHandoff: the data bytes must + // survive the slot migration). + void EnqueueRemoteFreeIfTracked(const TenantId& tenant_id, + const std::string& key, + QuotaEraseMode quota_mode); + // Lease id granted by the supervisor-owned CvmController. Reused by the + // SlotOwnerHeartbeat and segment owner records so they share the master + // registration lifecycle (auto-removed on lease expiry). 0 until the + // supervisor injects it via SetCvmLeaseId(). + EtcdLeaseId cvm_lease_id_{0}; + // 读路径 slot 所有权校验位图（A1）。由心跳 resolver 与晋升路径更新， + // GetReplicaList / BatchGetReplicaList 读路径校验，非 owner 拒绝服务。 + // owned_slots_ready_ 为 false 表示 partition 未启用或尚未解析过，放行。 + mutable std::shared_mutex owned_slots_mutex_; + std::vector owned_slot_lookup_; + bool owned_slots_ready_{false}; + // 仅当 owned slot 数量变化时打印 INFO 日志（避免心跳周期刷屏）。 + bool owned_slot_count_logged_{false}; + std::size_t last_logged_owned_count_{0}; + // Segment view (CVM) 数据源：在 segment 挂载/卸载时把 segment owner 原始 + // 记录同步到 etcd（segment_view/），供 CvmController 聚合生成 segment + // view 快照。仅在 etcd HA backend 下生效，其余场景为空操作。 + void PublishSegmentOwnerForCvm(const Segment& segment); + void RemoveSegmentOwnerForCvm(const UUID& segment_id); + // 动态 KV slot 划分：以 etcd 注册表为唯一事实源，读取已注册的 primary + // master 列表，按一致性哈希环（cvm::ResolveOwnedSlotsOnRing）计算本机 + // 应拥有的 slot 集合。etcd 读取失败时沿用上一轮结果（sticky），避免 + // 瞬时抖动引发全量抢夺；本机不在 primary 列表时返回空集合（不认领）。 + // 供 SlotOwnerHeartbeat 动态解析器回调调用（运行在心跳线程）。 + std::vector ResolveOwnedSlotsForCvm(); + // sticky 缓存：最近一次成功解析的 owned slot 集合（etcd 抖动时沿用）。 + std::vector cvm_last_resolved_owned_slots_; + mutable std::mutex cvm_resolver_mutex_; + // live primary → live primary 的 slot 元数据交接（P4 技术债 1）。 + // ExportSlotMetadata 把 `slot` 下所有对象的元数据序列化后写入 etcd，再 + // 从本地 metadata_shards_ 擦除；ImportSlotMetadata 从 etcd 读回并物化到 + // 本地 metadata_shards_。数据字节始终留在 segment，不搬移。两者均由 + // SlotOwnerHeartbeat 的 on_slot_released / on_slot_acquired 钩子在心跳 + // 线程调用。 + ErrorCode ExportSlotMetadata(uint16_t slot); + ErrorCode ImportSlotMetadata(uint16_t slot); + // A1：读路径 slot 所有权校验。UpdateOwnedSlots 由心跳 resolver 与晋升路径 + // 调用，把最新 owned slot 集合写入位图；OwnsSlot 供读路径查询（未就绪放行）。 + void UpdateOwnedSlots(const std::vector& slots); + bool OwnsSlot(uint16_t slot) const; + // 读路径转发：解析任意 slot 的 owner master_id（基于一致性哈希环与 + // etcd 中的 primary master 列表）。解析失败返回 nullopt。供 GetReplicaList + // / BatchGetReplicaList 在 OwnsSlot 失败时向 slot owner 转发请求。 + std::optional ResolveSlotOwnerMasterId(uint16_t slot) const; + // GetReplicaList 的本地查询核心：不含 slot 所有权校验与转发，供 + // client-facing GetReplicaList 与 InterMasterGetReplicaList 复用。 + tl::expected GetReplicaListLocal( + const ObjectIdentity& object_id); + // BatchGetReplicaList 的本地查询核心：不含 slot 所有权校验与转发，供 + // client-facing BatchGetReplicaList 与 InterMasterBatchGetReplicaList + // 复用。 + std::vector> + BatchGetReplicaListLocal(const std::vector& keys, + const TenantId& tenant_id); + bool OwnsVChunkSlot(uint16_t slot) const; // root filesystem directory for persistent storage const std::string root_fs_dir_; // global 3fs/nfs segment size @@ -2140,6 +2366,24 @@ class MasterService { // Segment management SegmentManager segment_manager_; NoFSegmentManager nof_segment_manager_; + VChunkMasterManager vchunk_manager_; + bool vchunk_enabled_{false}; + uint64_t vchunk_reaper_interval_ms_{1000}; + size_t vchunk_reaper_max_scan_{128}; + std::atomic vchunk_reaper_running_{false}; + bool vchunk_recovery_pending_{false}; + struct VChunkRemoteReadLease { + VChunkMasterManager::ReadHandle handle; + int64_t expires_at_ms{0}; + }; + std::mutex vchunk_read_leases_mutex_; + std::unordered_map + vchunk_read_leases_; + std::thread vchunk_reaper_thread_; + std::mutex vchunk_reaper_mutex_; + std::condition_variable vchunk_reaper_cv_; + void StartVChunkReaper(); + void VChunkReaperThreadFunc(); BufferAllocatorType memory_allocator_type_; const AllocationStrategyType allocation_strategy_type_; std::shared_ptr allocation_strategy_; diff --git a/mooncake-store/include/metadata_store.h b/mooncake-store/include/metadata_store.h index 97c886ddf7..a91f880046 100644 --- a/mooncake-store/include/metadata_store.h +++ b/mooncake-store/include/metadata_store.h @@ -42,6 +42,8 @@ struct StandbyObjectMetadata { // Check if this metadata has valid replicas bool HasReplicas() const { return !replicas.empty(); } }; +YLT_REFL(StandbyObjectMetadata, client_id, size, replicas, last_sequence_id, + group_id, data_type); /** * Segment info stored in standby's segment registry. @@ -72,6 +74,25 @@ struct StandbyObjectEntry { YLT_REFL(StandbyObjectEntry, tenant_id, key, metadata); }; +/** + * @brief Object-metadata export for a single KV slot (live primary -> live + * primary handoff). + * + * When a live primary gracefully releases a slot it no longer owns (scale-out / + * scale-in rebalance), it exports the object metadata of every key in that + * slot to etcd. The new owner then imports it to materialize the same + * key -> Replica::Descriptor[] mapping without moving any data bytes (which + * stay in their segments). Serialized with struct_pack (msgpack binary) and + * stored as a binary etcd value. + */ +struct SlotMetadataExport { + uint16_t slot{0}; + std::string source_master_id; + std::vector objects; + + YLT_REFL(SlotMetadataExport, slot, source_master_id, objects); +}; + /** * Complete snapshot exported from standby at promotion time. * Includes applied OpLog sequence ID, all object metadata, diff --git a/mooncake-store/include/partition/kv_hash_map.h b/mooncake-store/include/partition/kv_hash_map.h new file mode 100644 index 0000000000..65ad2f556b --- /dev/null +++ b/mooncake-store/include/partition/kv_hash_map.h @@ -0,0 +1,23 @@ +#pragma once + +#include +#include + +#include "cvm/slot_hash.h" +#include "tenant_id.h" + +namespace mooncake { +namespace partition { + +// client 侧哈希分片：key → 逻辑 slot（0..16383）。 +// 复用 cvm::KeySlot（CRC32C），保证与 submaster 内部 getSlot 使用同一份实现， +// 使 hash 相同的 key 必然落到同一个 submaster。 +class KvHashMap { + public: + static uint16_t Compute(const TenantId& tenant, const std::string& key) { + return cvm::KeySlot(tenant, key); + } +}; + +} // namespace partition +} // namespace mooncake diff --git a/mooncake-store/include/partition/partition_router.h b/mooncake-store/include/partition/partition_router.h new file mode 100644 index 0000000000..f2d212e57c --- /dev/null +++ b/mooncake-store/include/partition/partition_router.h @@ -0,0 +1,43 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "cvm/cvm_types.h" +#include "mutex.h" +#include "tenant_id.h" +#include "types.h" + +namespace mooncake { +namespace partition { + +// client 侧路由：应用 SlotOwner 映射，把逻辑 slot 解析为 submaster_id。 +// 映射来源为 etcd 快照（snapshot/kv_view，即 KvViewSnapshot），直读 etcd。 +class PartitionRouter { + public: + // 加载 slot → submaster 映射（覆盖式）。 + void LoadSlotOwners(const std::vector& owners); + + // 直读 etcd 快照（snapshot/kv_view）并刷新映射。 + ErrorCode LoadFromEtcdSnapshot(const std::string& cluster_namespace); + + // slot → submaster_id（primary_master_id）；未命中返回 nullopt。 + std::optional ResolveSubmaster(uint16_t slot) const; + + // key → submaster_id（先哈希再路由）；未命中返回 nullopt。 + std::optional Route(const TenantId& tenant, + const std::string& key) const; + + void Clear(); + size_t Size() const; + + private: + mutable SharedMutex mutex_; + std::unordered_map slot_to_submaster_; +}; + +} // namespace partition +} // namespace mooncake diff --git a/mooncake-store/include/rpc_service.h b/mooncake-store/include/rpc_service.h index dc38a091e2..52f062c4df 100644 --- a/mooncake-store/include/rpc_service.h +++ b/mooncake-store/include/rpc_service.h @@ -93,6 +93,24 @@ class WrappedMasterService { ReplicaType replica_type = ReplicaType::ALL, const std::string& tenant_id = "default"); + tl::expected VChunkPutStart( + const std::string& tenant_id, const std::string& key, + uint64_t total_size, int64_t now_ms); + tl::expected VChunkPutEnd( + const std::string& tenant_id, const std::string& key, + const std::string& vchunk_id, int64_t now_ms); + tl::expected VChunkPutRevoke( + const std::string& tenant_id, const std::string& key, + const std::string& vchunk_id); + tl::expected GetVChunk( + const std::string& tenant_id, const std::string& key); + tl::expected ReleaseVChunkReadLease( + const std::string& lease_id); + tl::expected RemoveVChunk(const std::string& tenant_id, + const std::string& key, + int64_t now_ms); + VChunkRuntimeInfo GetVChunkRuntimeInfo(); + std::vector, ErrorCode>> BatchPutStart(const UUID& client_id, const std::vector& keys, const std::vector& slice_lengths, @@ -219,6 +237,10 @@ class WrappedMasterService { tl::expected PollRemoveAll(const UUID& client_id); + tl::expected, ErrorCode> RemoveObjectHeartbeat( + const UUID& client_id); + tl::expected AckRemoveObjectHeartbeat( + const UUID& client_id, const std::vector& tasks); tl::expected ReportSsdCapacity( const UUID& client_id, int64_t ssd_total_capacity_bytes); @@ -263,6 +285,63 @@ class WrappedMasterService { uint64_t initial_oplog_sequence_id, const std::vector& segments); + // CVM slot ownership publishing, driven by the HA supervisor. These + // forward to the wrapped MasterService (NOT RPC endpoints). + void SetCvmLeaseId(EtcdLeaseId lease_id); + ErrorCode StartSlotOwnerHeartbeat(); + void StopSlotOwnerHeartbeat(); + + // CVM inter-master RPC lifecycle, driven by the HA supervisor (NOT an + // RPC endpoint). Starts/stops the peer discovery + connection pools. + ErrorCode StartInterMasterRpc(); + void StopInterMasterRpc(); + + // Inter-master handshake RPC: returns this submaster's identity and + // ownership summary so peers can verify the inter-master channel. + tl::expected + InterMasterHandshake(); + + // Inter-master allocation forwarding (CVM plan B): allocate memory + // replicas in this submaster's locally mounted segments on behalf of a + // slot-owning peer. Strictly within `preferred_segments` when given. + // This submaster keeps the real handles alive (keepalive registry); + // the slot owner materializes dummy-allocator replicas from the + // returned descriptors and owns the metadata. + tl::expected, ErrorCode> + InterMasterAllocateReplicas( + const std::string& tenant_id, const std::string& key, + uint64_t slice_length, uint64_t replica_num, + const std::vector& preferred_segments); + + // Frees the keepalive entry (and thus the handles) previously created + // by InterMasterAllocateReplicas. Returns true when an entry existed. + tl::expected InterMasterFreeReplicas( + const std::string& tenant_id, const std::string& key); + + // Inter-master read forwarding (CVM plan B): local GetReplicaList / + // BatchGetReplicaList query for a forwarding peer. These do NOT + // re-forward, so an inconsistent view terminates the chain at the first + // hop instead of looping. + tl::expected InterMasterGetReplicaList( + const std::string& key, const std::string& tenant_id); + + std::vector> + InterMasterBatchGetReplicaList(const std::vector& keys, + const std::string& tenant_id); + + // Inter-master write forwarding (model B): relays a FULL PutStart to the + // slot-owning submaster so it allocates + writes metadata locally. + tl::expected, ErrorCode> InterMasterPutStart( + const UUID& client_id, const std::string& key, + const std::string& tenant_id, uint64_t slice_length, + const ReplicateConfig& config); + + // Upsert variant of InterMasterPutStart (owner overwrites-if-exists). + tl::expected, ErrorCode> + InterMasterUpsertStart(const UUID& client_id, const std::string& key, + const std::string& tenant_id, uint64_t slice_length, + const ReplicateConfig& config); + tl::expected CreateCopyTask( const std::string& key, const std::string& tenant_id, const std::vector& targets); diff --git a/mooncake-store/include/rpc_types.h b/mooncake-store/include/rpc_types.h index e9c6438734..9a43629882 100644 --- a/mooncake-store/include/rpc_types.h +++ b/mooncake-store/include/rpc_types.h @@ -122,6 +122,22 @@ struct MoveStartResponse { }; YLT_REFL(MoveStartResponse, source, target); +/** + * @brief Response structure for InterMasterHandshake: identity + ownership + * summary of a submaster, used by other submasters to verify the inter-master + * RPC channel (CVM multi-submaster coordination, forwarding path). + */ +struct InterMasterHandshakeResponse { + std::string master_id; + uint64_t lease_id{0}; + uint32_t owned_slot_count{0}; + std::string version; + + InterMasterHandshakeResponse() = default; +}; +YLT_REFL(InterMasterHandshakeResponse, master_id, lease_id, owned_slot_count, + version); + enum class JobType { DRAIN = 0, }; diff --git a/mooncake-store/include/segment.h b/mooncake-store/include/segment.h index 75b9b0c0f4..0fc34a841a 100644 --- a/mooncake-store/include/segment.h +++ b/mooncake-store/include/segment.h @@ -100,6 +100,12 @@ struct LocalDiskSegment { // offloading_objects (offloading_mutex_). std::unordered_map GUARDED_BY( offloading_mutex_) promotion_objects; + // Keys removed via Remove/BatchRemove that had LOCAL_DISK replicas on + // this client. Populated by master's Remove when the key has a + // LOCAL_DISK replica. Drained by RemoveObjectHeartbeat RPC. Same locking as + // offloading_objects (offloading_mutex_). + std::vector GUARDED_BY( + offloading_mutex_) removed_keys; // Set by master's RemoveAll. When the client sees this flag via // PollRemoveAll, it calls FileStorage::RemoveAll() to physically // delete all SSD files. Same locking as offloading_objects diff --git a/mooncake-store/include/storage_backend.h b/mooncake-store/include/storage_backend.h index d9863896d0..b417c3c950 100644 --- a/mooncake-store/include/storage_backend.h +++ b/mooncake-store/include/storage_backend.h @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -12,6 +13,7 @@ #include #include #include +#include #include #include @@ -40,6 +42,10 @@ struct BucketMetadata { std::vector keys; std::vector metadatas; + // Persisted tombstones. Init filters these keys before rebuilding the + // in-memory object index. + std::vector tombstones; + // Runtime-only fields (not serialized) for safe deletion support // Tracks number of in-flight reads to enable safe bucket deletion mutable std::atomic inflight_reads_{0}; @@ -47,6 +53,17 @@ struct BucketMetadata { // Updated on every read with relaxed ordering (approximate is sufficient). mutable std::atomic last_access_ns_{0}; + // Runtime-only (not serialized): bytes marked removed via MarkRemoved. + // Drives GC candidate selection (compact when deleted_bytes_ > 0). + mutable std::atomic deleted_bytes_{0}; + // Runtime-only (not serialized): true while a GC compaction is in flight + // for this bucket, preventing re-entrant compaction. + mutable std::atomic compacting_{false}; + // Runtime-only version of the bucket's deletion state. Compaction captures + // this value with its read snapshot and validates it before publishing a + // new bucket, so a concurrent deletion cannot publish stale data. + uint64_t generation_{0}; + // Default constructor BucketMetadata() = default; @@ -56,8 +73,12 @@ struct BucketMetadata { data_size(other.data_size), keys(other.keys), metadatas(other.metadatas), + tombstones(other.tombstones), inflight_reads_(0), - last_access_ns_(0) {} + last_access_ns_(0), + deleted_bytes_(0), + compacting_(false), + generation_(0) {} // Move constructor BucketMetadata(BucketMetadata&& other) noexcept @@ -65,8 +86,12 @@ struct BucketMetadata { data_size(other.data_size), keys(std::move(other.keys)), metadatas(std::move(other.metadatas)), + tombstones(std::move(other.tombstones)), inflight_reads_(0), - last_access_ns_(0) {} + last_access_ns_(0), + deleted_bytes_(0), + compacting_(false), + generation_(0) {} // Copy assignment BucketMetadata& operator=(const BucketMetadata& other) { @@ -75,6 +100,7 @@ struct BucketMetadata { data_size = other.data_size; keys = other.keys; metadatas = other.metadatas; + tombstones = other.tombstones; // Don't copy runtime state } return *this; @@ -87,12 +113,13 @@ struct BucketMetadata { data_size = other.data_size; keys = std::move(other.keys); metadatas = std::move(other.metadatas); + tombstones = std::move(other.tombstones); // Don't move runtime state } return *this; } }; -YLT_REFL(BucketMetadata, data_size, keys, metadatas); +YLT_REFL(BucketMetadata, data_size, keys, metadatas, tombstones); /** * @brief RAII guard for tracking in-flight bucket reads. @@ -204,6 +231,22 @@ struct BucketBackendConfig { // eviction_policy. Set via // MOONCAKE_OFFLOAD_DISABLE_SSD_EVICTION. + // --- Explicit-delete-only GC config --- + // Enable background tombstone compaction GC. + bool gc_enable = true; + // GC scan interval in milliseconds. + int64_t gc_interval_ms = 1000; + // Compact a bucket when deleted bytes / bucket data size >= this ratio. + double gc_deleted_ratio = 0.25; + // Trigger GC when total_size / max_total_size >= this ratio. + double gc_high_watermark_ratio = 0.90; + // Max old buckets collected per GC round for cross-bucket merge. + int64_t gc_max_buckets_per_round = 1; + // Enable cross-bucket merge compaction (collect live keys from multiple + // tombstone buckets into one new bucket). When false, each bucket is + // compacted independently (no merge). + bool gc_merge_enable = true; + bool Validate() const; static BucketBackendConfig FromEnvironment(); @@ -412,6 +455,20 @@ class StorageBackendInterface { // Default: no-op (no test failures injected) } + // Mark a key as removed (tombstone) for explicit-delete-only GC. + // Default no-op: only BucketStorageBackend implements tombstone + GC. + // File-per-key and other backends inherit the no-op (do not delete files). + // Safe to call for keys not present in local storage (idempotent). + virtual tl::expected MarkRemoved( + const std::string& /* key */) { + return {}; + } + + // Batch variant: mark multiple keys as removed in one lock acquisition. + virtual tl::expected BatchMarkRemoved( + const std::vector& /* keys */) { + return {}; + } // Remove all persisted objects from disk. Called during RemoveAll to // clean up physical SSD files alongside master metadata deletion. virtual void RemoveAll() {} @@ -1005,11 +1062,53 @@ class BucketStorageBackend : public StorageBackendInterface { */ tl::expected DeleteBucket(int64_t bucket_id); + // Explicit-delete-only GC: mark a key as tombstone (no disk IO). + // Removes key from object_bucket_map_ (immediately invisible to + // BatchLoad/IsExist) and bumps bucket deleted_bytes_. + // Idempotent: no-op if key not in local storage. + tl::expected MarkRemoved( + const std::string& key) override; + tl::expected BatchMarkRemoved( + const std::vector& keys) override; + + // Compact a single bucket: copy-on-write live keys to a new bucket, + // atomically swap mappings, delete old bucket file after reads drain. + // Returns true on success (or no-op), false on transient failure + // (will retry next round). Public to allow explicit compaction and + // testing (analogous to DeleteBucket). + bool CompactBucket(int64_t bucket_id); + + // Compact multiple buckets into one new bucket (cross-bucket merge). + // Collects live keys from all given old buckets, groups them by + // bucket_keys_limit/bucket_size_limit, and writes ONE new bucket per + // round (the first group that fills up). If the first group doesn't + // fill a full bucket and there's no space pressure, the merge is + // deferred to the next round. Old buckets whose live keys are all + // migrated are deleted. + // Returns true on success (or deferred), false on transient failure. + bool CompactBuckets(const std::vector& bucket_ids, + bool space_pressure = false); tl::expected, ErrorCode> EvictAboveDiskWatermark( double high_watermark_ratio, double low_watermark_ratio, EvictionHandler eviction_handler = nullptr) override; private: + // --- Background GC --- + // Background GC thread entry point. + void GCThreadFunc(); + + // Wait for in-flight reads on a bucket to drain (up to 10s). + void WaitForInflightReads(std::shared_ptr bucket); + + // Delete .bucket and .meta files for a bucket_id, ignore missing. + void DeleteBucketFiles(int64_t bucket_id); + + // GC thread lifecycle members + std::atomic gc_running_{false}; + std::thread gc_thread_; + std::mutex gc_mutex_; + std::condition_variable gc_cv_; + tl::expected, ErrorCode> BuildBucket( int64_t bucket_id, const std::unordered_map>& batch_object, diff --git a/mooncake-store/include/types.h b/mooncake-store/include/types.h index b7c983786d..e4c67adc94 100644 --- a/mooncake-store/include/types.h +++ b/mooncake-store/include/types.h @@ -258,6 +258,16 @@ struct PromotionTaskItem { }; YLT_REFL(PromotionTaskItem, tenant_id, key, size); +struct RemoveTaskItem { + std::string tenant_id; + std::string key; + + bool operator==(const RemoveTaskItem& other) const { + return tenant_id == other.tenant_id && key == other.key; + } +}; +YLT_REFL(RemoveTaskItem, tenant_id, key); + // Store client configuration validation limits static constexpr size_t MIN_SEGMENT_SIZE = 1024; // 1KB static constexpr size_t MAX_SEGMENT_SIZE = 1024ULL * 1024 * 1024 * 1024; // 1TB @@ -365,6 +375,9 @@ enum class ErrorCode : int32_t { -1010, ///< Request cannot be done in current status. UNAVAILABLE_IN_CURRENT_MODE = -1011, ///< Request cannot be done in current mode. + SLOT_NOT_OWNED = + -1012, ///< The key's slot is not owned by this master (KV partition + ///< rebalanced); the client should re-route. // FILE errors (Range: -1100 to -1199) FILE_NOT_FOUND = -1100, ///< File not found. diff --git a/mooncake-store/include/vchunk_allocation_strategy.h b/mooncake-store/include/vchunk_allocation_strategy.h new file mode 100644 index 0000000000..2d7a845039 --- /dev/null +++ b/mooncake-store/include/vchunk_allocation_strategy.h @@ -0,0 +1,52 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +#include "allocator.h" +#include "vchunk_config.h" + +namespace mooncake { + +class AllocatorManager; + +struct VCSliceAllocation { + uint32_t slice_index{0}; + std::string segment_name; + uint64_t target_offset{0}; + uint32_t logical_length{0}; + uint32_t allocated_length{0}; + std::unique_ptr buffer; + + VCSliceAllocation() = default; + VCSliceAllocation(VCSliceAllocation&&) noexcept = default; + VCSliceAllocation& operator=(VCSliceAllocation&&) noexcept = default; + VCSliceAllocation(const VCSliceAllocation&) = delete; + VCSliceAllocation& operator=(const VCSliceAllocation&) = delete; +}; + +class VChunkAllocationResult { + public: + VChunkAllocationResult() = default; + VChunkAllocationResult(VChunkAllocationResult&&) noexcept = default; + VChunkAllocationResult& operator=(VChunkAllocationResult&&) noexcept = + default; + VChunkAllocationResult(const VChunkAllocationResult&) = delete; + VChunkAllocationResult& operator=(const VChunkAllocationResult&) = delete; + + size_t row_size{0}; + std::vector allocations; +}; + +tl::expected AllocateVChunk( + const AllocatorManager& allocator_manager, uint64_t total_size, + VCSliceSizeLevel slice_size_level, + const std::set& excluded_segments = {}); + +} // namespace mooncake diff --git a/mooncake-store/include/vchunk_client.h b/mooncake-store/include/vchunk_client.h new file mode 100644 index 0000000000..a7b70988c5 --- /dev/null +++ b/mooncake-store/include/vchunk_client.h @@ -0,0 +1,99 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "tenant_id.h" +#include "types.h" +#include "vchunk_control_plane.h" +#include "vchunk_metadata.h" +#include "vchunk_metrics.h" + +namespace mooncake { + +class VChunkDataPlane { + public: + virtual ~VChunkDataPlane() = default; + virtual ErrorCode Write(const VChunkMetadataRecord& record, + const void* source, size_t length, + std::chrono::steady_clock::time_point deadline) = 0; + virtual ErrorCode Read(const VChunkMetadataRecord& record, void* destination, + size_t length, + std::chrono::steady_clock::time_point deadline) = 0; +}; + +class VChunkLegacyPath { + public: + virtual ~VChunkLegacyPath() = default; + virtual ErrorCode Put(const TenantId&, const std::string&, const void*, + size_t) = 0; + virtual ErrorCode Get(const TenantId&, const std::string&, void*, size_t) = 0; + virtual ErrorCode Remove(const TenantId&, const std::string&) = 0; +}; + +class VChunkClient { + public: + using Clock = std::chrono::steady_clock; + using NowMs = std::function; + + VChunkClient(bool enabled, MasterService& master, VChunkDataPlane& data_plane, + VChunkLegacyPath& legacy, std::chrono::milliseconds timeout, + NowMs now_ms, + uint32_t max_retries = VChunkConfig{}.max_slice_retry, + uint32_t circuit_breaker_threshold = 0, + std::shared_ptr metrics = nullptr); + VChunkClient(bool enabled, VChunkControlPlane& control_plane, + VChunkDataPlane& data_plane, VChunkLegacyPath& legacy, + std::chrono::milliseconds timeout, NowMs now_ms, + uint32_t max_retries = VChunkConfig{}.max_slice_retry, + uint32_t circuit_breaker_threshold = 0, + std::shared_ptr metrics = nullptr); + + ErrorCode Put(const TenantId& tenant_id, const std::string& key, + const void* source, size_t length); + ErrorCode Get(const TenantId& tenant_id, const std::string& key, + void* destination, size_t length); + ErrorCode Remove(const TenantId& tenant_id, const std::string& key); + + struct PutRequest { + std::string key; + const void* source{nullptr}; + size_t length{0}; + }; + struct GetRequest { + std::string key; + void* destination{nullptr}; + size_t length{0}; + }; + + std::vector BatchPut(const TenantId& tenant_id, + const std::vector& requests); + std::vector BatchGet(const TenantId& tenant_id, + const std::vector& requests); + std::vector BatchRemove( + const TenantId& tenant_id, const std::vector& keys); + VChunkMetricsSnapshot MetricsSnapshot() const; + + private: + bool enabled_; + std::unique_ptr owned_control_plane_; + VChunkControlPlane* control_plane_; + VChunkDataPlane& data_plane_; + VChunkLegacyPath& legacy_; + std::chrono::milliseconds timeout_; + NowMs now_ms_; + uint32_t max_retries_; + uint32_t circuit_breaker_threshold_; + std::atomic consecutive_put_failures_{0}; + std::shared_ptr metrics_; +}; + +} // namespace mooncake diff --git a/mooncake-store/include/vchunk_config.h b/mooncake-store/include/vchunk_config.h new file mode 100644 index 0000000000..a7569a5cfd --- /dev/null +++ b/mooncake-store/include/vchunk_config.h @@ -0,0 +1,42 @@ +#pragma once + +#include + +#include "types.h" + +namespace mooncake { + +enum class VCSliceSizeLevel : uint32_t { + k4K = 4U * 1024U, + k64K = 64U * 1024U, + k256K = 256U * 1024U, + k1M = 1024U * 1024U, +}; + +constexpr uint32_t SliceSizeLevelToBytes(VCSliceSizeLevel level) { + return static_cast(level); +} + +VCSliceSizeLevel SelectVChunkSliceSize(uint64_t value_size, + bool is_ssd_segment); + +struct VChunkConfig { + bool enabled{false}; + uint64_t creating_timeout_ms{30'000}; + uint64_t releasing_timeout_ms{60'000}; + uint32_t max_slice_retry{3}; + uint32_t max_slice_count{4096}; + uint64_t max_metadata_bytes{1024U * 1024U}; + uint32_t max_creating_objects{1024}; + uint64_t reaper_interval_ms{1000}; + uint32_t reaper_max_scan{128}; + + ErrorCode Validate() const; + + YLT_REFL(VChunkConfig, enabled, creating_timeout_ms, + releasing_timeout_ms, max_slice_retry, max_slice_count, + max_metadata_bytes, max_creating_objects, reaper_interval_ms, + reaper_max_scan); +}; + +} // namespace mooncake diff --git a/mooncake-store/include/vchunk_control_plane.h b/mooncake-store/include/vchunk_control_plane.h new file mode 100644 index 0000000000..797d40279a --- /dev/null +++ b/mooncake-store/include/vchunk_control_plane.h @@ -0,0 +1,80 @@ +#pragma once + +#include +#include +#include + +#include + +#include "tenant_id.h" +#include "types.h" +#include "vchunk_metadata.h" + +namespace mooncake { + +class MasterClient; +class MasterService; + +struct VChunkControlPlaneRead { + VChunkMetadataRecord record; + // Retains either the local allocator-backed read handle or the guard for a + // remote read lease until the data-plane transfer has finished. + std::shared_ptr lifetime; +}; + +class VChunkControlPlane { + public: + virtual ~VChunkControlPlane() = default; + + virtual tl::expected PutStart( + const TenantId& tenant_id, const std::string& key, uint64_t total_size, + int64_t now_ms) = 0; + virtual ErrorCode PutEnd(const TenantId& tenant_id, const std::string& key, + const std::string& vchunk_id, + int64_t now_ms) = 0; + virtual ErrorCode PutRevoke(const TenantId& tenant_id, + const std::string& key, + const std::string& vchunk_id) = 0; + virtual tl::expected Get( + const TenantId& tenant_id, const std::string& key) = 0; + virtual ErrorCode Remove(const TenantId& tenant_id, + const std::string& key, int64_t now_ms) = 0; +}; + +class LocalVChunkControlPlane final : public VChunkControlPlane { + public: + explicit LocalVChunkControlPlane(MasterService& master) : master_(master) {} + + tl::expected PutStart( + const TenantId&, const std::string&, uint64_t, int64_t) override; + ErrorCode PutEnd(const TenantId&, const std::string&, const std::string&, + int64_t) override; + ErrorCode PutRevoke(const TenantId&, const std::string&, + const std::string&) override; + tl::expected Get( + const TenantId&, const std::string&) override; + ErrorCode Remove(const TenantId&, const std::string&, int64_t) override; + + private: + MasterService& master_; +}; + +class RpcVChunkControlPlane final : public VChunkControlPlane { + public: + explicit RpcVChunkControlPlane(MasterClient& master) : master_(master) {} + + tl::expected PutStart( + const TenantId&, const std::string&, uint64_t, int64_t) override; + ErrorCode PutEnd(const TenantId&, const std::string&, const std::string&, + int64_t) override; + ErrorCode PutRevoke(const TenantId&, const std::string&, + const std::string&) override; + tl::expected Get( + const TenantId&, const std::string&) override; + ErrorCode Remove(const TenantId&, const std::string&, int64_t) override; + + private: + MasterClient& master_; +}; + +} // namespace mooncake diff --git a/mooncake-store/include/vchunk_master_manager.h b/mooncake-store/include/vchunk_master_manager.h new file mode 100644 index 0000000000..f7d5117c83 --- /dev/null +++ b/mooncake-store/include/vchunk_master_manager.h @@ -0,0 +1,100 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "allocator.h" +#include "tenant_id.h" +#include "vchunk_allocation_strategy.h" +#include "vchunk_config.h" +#include "vchunk_metadata.h" +#include "vchunk_metadata_store.h" +#include "vchunk_metrics.h" + +namespace mooncake { + +// In-memory master-side vchunk lifecycle manager. The caller must hold the +// SegmentManager allocator access guard while PutStart uses AllocatorManager. +class VChunkMasterManager { + public: + using OwnershipPredicate = + std::function; + + class ReadHandle { + public: + ReadHandle() = default; + const VChunkMetadataRecord& record() const { return record_; } + + private: + friend class VChunkMasterManager; + VChunkMetadataRecord record_; + std::shared_ptr lifetime_; + }; + + explicit VChunkMasterManager( + VChunkConfig config, + std::shared_ptr metadata_store = nullptr, + std::shared_ptr metrics = nullptr); + + VChunkMasterManager(const VChunkMasterManager&) = delete; + VChunkMasterManager& operator=(const VChunkMasterManager&) = delete; + + tl::expected PutStart( + const AllocatorManager& allocator_manager, const TenantId& tenant_id, + const std::string& key, uint64_t total_size, bool is_ssd_segment, + int64_t now_ms, + const std::set& excluded_segments = {}); + + ErrorCode PutEnd(const TenantId& tenant_id, const std::string& key, + const std::string& vchunk_id, int64_t now_ms); + ErrorCode PutRevoke(const TenantId& tenant_id, const std::string& key, + const std::string& vchunk_id); + + tl::expected Get( + const TenantId& tenant_id, const std::string& key) const; + tl::expected AcquireRead( + const TenantId& tenant_id, const std::string& key) const; + + ErrorCode Remove(const TenantId& tenant_id, const std::string& key, + int64_t now_ms); + + ErrorCode Recover(int64_t now_ms, OwnershipPredicate owns = {}); + tl::expected ReapExpired(int64_t now_ms, + size_t max_scan, + OwnershipPredicate owns = {}); + VChunkMetricsSnapshot MetricsSnapshot() const; + + size_t SizeForTesting() const; + bool HasPersistentMetadata() const { return metadata_store_->IsPersistent(); } + + private: + struct Entry { + VChunkMetadataRecord record; + std::vector> buffers; + }; + + static std::string ScopedKey(const TenantId& tenant_id, + const std::string& key); + void RefreshStateMetricsLocked(); + void ReleasePendingPut(const std::string& scoped_key); + + const VChunkConfig config_; + const std::shared_ptr metadata_store_; + const std::shared_ptr metrics_; + mutable std::mutex mutex_; + std::unordered_map> entries_; + std::unordered_set pending_puts_; + std::string reaper_cursor_key_; +}; + +} // namespace mooncake diff --git a/mooncake-store/include/vchunk_metadata.h b/mooncake-store/include/vchunk_metadata.h new file mode 100644 index 0000000000..cd0d3093fb --- /dev/null +++ b/mooncake-store/include/vchunk_metadata.h @@ -0,0 +1,104 @@ +#pragma once + +#include +#include +#include + +#include + +#include "mutex.h" +#include "types.h" +#include "vchunk_config.h" + +namespace mooncake { + +inline constexpr uint32_t kVChunkMetadataSchemaVersion = 1; + +enum class VCSliceStatus : uint8_t { + PENDING = 0, + COMPLETED = 1, + FAILED = 2, +}; + +enum class VChunkStatus : uint8_t { + CREATING = 0, + ACTIVE = 1, + RELEASING = 2, + RELEASED = 3, + FAILED = 4, +}; + +struct VCSliceDescriptor { + uint32_t slice_index{0}; + std::string target_segment_name; + uint64_t target_offset{0}; + uint32_t logical_length{0}; + uint32_t allocated_length{0}; + VCSliceStatus status{VCSliceStatus::PENDING}; + uint32_t retry_count{0}; + + YLT_REFL(VCSliceDescriptor, slice_index, target_segment_name, + target_offset, logical_length, allocated_length, status, + retry_count); +}; + +// Stable wire/storage representation. Runtime-only synchronization and indexes +// intentionally live outside this type. +struct VChunkMetadataRecord { + uint32_t schema_version{kVChunkMetadataSchemaVersion}; + std::string vchunk_id; + std::string tenant_id; + std::string key; + uint64_t total_size{0}; + uint32_t slice_count{0}; + VCSliceSizeLevel slice_size_level{VCSliceSizeLevel::k4K}; + std::vector slices; + uint32_t row_size{0}; + VChunkStatus status{VChunkStatus::CREATING}; + int64_t created_at_ms{0}; + int64_t last_updated_at_ms{0}; + + YLT_REFL(VChunkMetadataRecord, schema_version, vchunk_id, tenant_id, key, + total_size, slice_count, slice_size_level, slices, row_size, + status, created_at_ms, last_updated_at_ms); +}; + +struct VChunkRuntimeInfo { + bool enabled{false}; + bool persistent_metadata{false}; + + YLT_REFL(VChunkRuntimeInfo, enabled, persistent_metadata); +}; + +struct VChunkReadLease { + VChunkMetadataRecord record; + std::string lease_id; + + YLT_REFL(VChunkReadLease, record, lease_id); +}; + +ErrorCode ValidateVChunkMetadata(const VChunkMetadataRecord& record, + const VChunkConfig& config); +ErrorCode ValidateVChunkTransition(VChunkStatus from, VChunkStatus to); + +tl::expected, ErrorCode> SerializeVChunkMetadata( + const VChunkMetadataRecord& record, const VChunkConfig& config); +tl::expected DeserializeVChunkMetadata( + const std::vector& bytes, const VChunkConfig& config); + +class VChunkMetadata { + public: + explicit VChunkMetadata(VChunkMetadataRecord record); + + VChunkMetadata(const VChunkMetadata&) = delete; + VChunkMetadata& operator=(const VChunkMetadata&) = delete; + + VChunkMetadataRecord Snapshot() const; + ErrorCode TransitionTo(VChunkStatus next, int64_t updated_at_ms); + + private: + mutable SpinLock lock_; + VChunkMetadataRecord record_; +}; + +} // namespace mooncake diff --git a/mooncake-store/include/vchunk_metadata_store.h b/mooncake-store/include/vchunk_metadata_store.h new file mode 100644 index 0000000000..c7c0e6e32d --- /dev/null +++ b/mooncake-store/include/vchunk_metadata_store.h @@ -0,0 +1,63 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include "types.h" +#include "vchunk_metadata.h" + +namespace mooncake { + +inline constexpr char kVChunkMetadataNamespace[] = "/mooncake/vchunk/v1"; + +std::string MakeVChunkMetadataStoreKey(const VChunkMetadataRecord& record); + +class VChunkMetadataStore { + public: + virtual ~VChunkMetadataStore() = default; + virtual ErrorCode Put(const VChunkMetadataRecord& record) = 0; + virtual ErrorCode Remove(const VChunkMetadataRecord& record) = 0; + virtual tl::expected, ErrorCode> List() = 0; + virtual bool IsPersistent() const = 0; +}; + +class InMemoryVChunkMetadataStore final : public VChunkMetadataStore { + public: + ErrorCode Put(const VChunkMetadataRecord& record) override; + ErrorCode Remove(const VChunkMetadataRecord& record) override; + tl::expected, ErrorCode> List() override; + bool IsPersistent() const override { return false; } + + private: + std::mutex mutex_; + std::unordered_map records_; +}; + +// Persistent metadata backend for distributed validation. A scoped object +// index is stored separately from the vchunk-id record so concurrent masters +// cannot both create the same logical object. +class EtcdVChunkMetadataStore final : public VChunkMetadataStore { + public: + EtcdVChunkMetadataStore(std::string endpoints, VChunkConfig config, + std::string cluster_id = "default"); + + ErrorCode Put(const VChunkMetadataRecord& record) override; + ErrorCode Remove(const VChunkMetadataRecord& record) override; + tl::expected, ErrorCode> List() override; + bool IsPersistent() const override { return true; } + + ErrorCode connection_error() const { return connection_error_; } + + private: + std::string endpoints_; + VChunkConfig config_; + std::string namespace_prefix_; + ErrorCode connection_error_{ErrorCode::OK}; +}; + +} // namespace mooncake diff --git a/mooncake-store/include/vchunk_metrics.h b/mooncake-store/include/vchunk_metrics.h new file mode 100644 index 0000000000..a6150a5bb3 --- /dev/null +++ b/mooncake-store/include/vchunk_metrics.h @@ -0,0 +1,63 @@ +#pragma once + +#include +#include +#include +#include + +#include "vchunk_metadata.h" + +namespace mooncake { + +enum class VChunkOperation : uint8_t { PUT = 0, GET = 1, REMOVE = 2 }; + +struct VChunkMetricsSnapshot { + std::array requests{}; + std::array successes{}; + std::array latency_us{}; + uint64_t slices{0}; + uint64_t segment_participations{0}; + std::array slice_size_distribution{}; + uint64_t allocation_failures{0}; + uint64_t transfer_failures{0}; + uint64_t timeouts{0}; + uint64_t retries{0}; + uint64_t rollbacks{0}; + uint64_t metadata_bytes{0}; + uint64_t allocated_bytes{0}; + std::array states{}; +}; + +class VChunkMetrics { + public: + void Observe(VChunkOperation operation, bool success, uint64_t latency_us); + void AddSlices(uint64_t count) { slices_.fetch_add(count); } + void ObserveLayout(const VChunkMetadataRecord& record); + void AddAllocationFailure() { ++allocation_failures_; } + void AddTransferFailure() { ++transfer_failures_; } + void AddTimeout() { ++timeouts_; } + void AddRetry() { ++retries_; } + void AddRollback() { ++rollbacks_; } + void AddMetadataBytes(uint64_t bytes) { metadata_bytes_.fetch_add(bytes); } + void SetStateCount(VChunkStatus state, uint64_t count); + void SetAllocatedBytes(uint64_t bytes) { allocated_bytes_.store(bytes); } + VChunkMetricsSnapshot Snapshot() const; + + private: + std::array, 3> requests_{}; + std::array, 3> successes_{}; + std::array, 3> latency_us_{}; + std::atomic slices_{0}; + std::atomic segment_participations_{0}; + std::array, 4> slice_size_distribution_{}; + std::atomic allocation_failures_{0}; + std::atomic transfer_failures_{0}; + std::atomic timeouts_{0}; + std::atomic retries_{0}; + std::atomic rollbacks_{0}; + std::atomic metadata_bytes_{0}; + std::atomic allocated_bytes_{0}; + std::array, 5> states_{}; +}; + +} // namespace mooncake diff --git a/mooncake-store/include/vchunk_transfer_engine.h b/mooncake-store/include/vchunk_transfer_engine.h new file mode 100644 index 0000000000..411bd7688a --- /dev/null +++ b/mooncake-store/include/vchunk_transfer_engine.h @@ -0,0 +1,40 @@ +#pragma once + +#include +#include +#include + +#include "transfer_engine.h" +#include "vchunk_client.h" + +namespace mooncake { + +using VChunkSegmentResolver = + std::function(const std::string&)>; + +tl::expected, ErrorCode> +BuildVChunkTransferRequests(const VChunkMetadataRecord& record, void* buffer, + size_t length, TransferRequest::OpCode opcode, + const VChunkSegmentResolver& resolve_segment); + +class TransferEngineVChunkDataPlane final : public VChunkDataPlane { + public: + explicit TransferEngineVChunkDataPlane(TransferEngine& engine) + : engine_(engine) {} + + ErrorCode Write(const VChunkMetadataRecord& record, const void* source, + size_t length, + std::chrono::steady_clock::time_point deadline) override; + ErrorCode Read(const VChunkMetadataRecord& record, void* destination, + size_t length, + std::chrono::steady_clock::time_point deadline) override; + + private: + ErrorCode Transfer(const VChunkMetadataRecord& record, void* buffer, + size_t length, TransferRequest::OpCode opcode, + std::chrono::steady_clock::time_point deadline); + + TransferEngine& engine_; +}; + +} // namespace mooncake diff --git a/mooncake-store/src/CMakeLists.txt b/mooncake-store/src/CMakeLists.txt index 3031dbda86..41d3392057 100644 --- a/mooncake-store/src/CMakeLists.txt +++ b/mooncake-store/src/CMakeLists.txt @@ -16,6 +16,13 @@ set(MOONCAKE_STORE_SOURCES storage_backend.cpp thread_pool.cpp etcd_helper.cpp + cvm/etcd_view_store.cpp + cvm/cvm_controller.cpp + cvm/slot_owner_heartbeat.cpp + cvm/slot_migrator.cpp + cvm/cvm_http_server.cpp + cvm/inter_master_rpc.cpp + partition/partition_router.cpp segment.cpp transfer_task.cpp tenant_quota.cpp @@ -75,6 +82,15 @@ set(MOONCAKE_STORE_SOURCES ha_metric_manager.cpp store_c.cpp memory_alloc.cpp + vchunk_config.cpp + vchunk_metadata.cpp + vchunk_allocation_strategy.cpp + vchunk_master_manager.cpp + vchunk_client.cpp + vchunk_control_plane.cpp + vchunk_transfer_engine.cpp + vchunk_metadata_store.cpp + vchunk_metrics.cpp ssd_register_client.cpp engram/engram_store.cpp) diff --git a/mooncake-store/src/client_service.cpp b/mooncake-store/src/client_service.cpp index 501e5567fc..d25638cc1a 100644 --- a/mooncake-store/src/client_service.cpp +++ b/mooncake-store/src/client_service.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #ifdef USE_NOF #include #endif @@ -34,6 +35,10 @@ #include "transport/transport.h" #include "config.h" #include "ha/leadership/leader_coordinator_factory.h" +#include "cvm/cvm_keys.h" +#include "cvm/cvm_types.h" +#include "cvm/etcd_view_store.h" +#include "etcd_helper.h" #include "types.h" #include "client_buffer.h" #include "utils.h" @@ -44,6 +49,12 @@ #include "environ.h" #include "mooncake_logging.h" +#if __has_include() +#include +#else +#include +#endif + #define SPDIAG_PERF_DEF_FILE "mooncake_perf_points.def" #define SPDIAG_PROGRAM_NAME "mooncake_store" #include "spdiag/auto_perf.h" @@ -347,6 +358,11 @@ Client::~Client() { leader_monitor_thread_.join(); } + routing_refresh_running_ = false; + if (routing_refresh_thread_.joinable()) { + routing_refresh_thread_.join(); + } + { std::lock_guard lock(graceful_unmount_timer_mutex_); graceful_unmount_timer_stopping_ = true; @@ -557,8 +573,24 @@ ErrorCode Client::ConnectToMaster(const std::string& master_server_entry) { return current_view.error(); } if (!current_view.value().has_value()) { - LOG(ERROR) << "No master is available in HA backend"; - return ErrorCode::UNAVAILABLE_IN_CURRENT_STATUS; + // No single-leader master_view is published. In the CVM + // multi-submaster architecture masters never write master_view; + // they register under /cvm//masters/ instead. Fall back to + // the CVM registry to bootstrap the connection. Per-key routing + // is loaded afterwards via LoadPartitionRouting() from the + // kv_view snapshot in etcd. + LOG(INFO) << "No master_view in HA backend; trying CVM " + "multi-submaster bootstrap from /cvm/ registry"; + auto err = + ConnectToCvmSubmasters(ha_backend_spec.value()->connstring); + if (err != ErrorCode::OK) { + LOG(ERROR) << "No master is available in HA backend (nor a " + "CVM submaster registry)"; + return err; + } + direct_master_address_.clear(); + etcd_connstring_ = ha_backend_spec.value()->connstring; + return ErrorCode::OK; } const auto& master_view = current_view.value().value(); @@ -570,6 +602,7 @@ ErrorCode Client::ConnectToMaster(const std::string& master_server_entry) { leader_coordinator_ = std::move(coordinator.value()); direct_master_address_.clear(); + etcd_connstring_ = ha_backend_spec.value()->connstring; leader_monitor_running_ = true; leader_monitor_thread_ = @@ -591,6 +624,264 @@ ErrorCode Client::ConnectToMaster(const std::string& master_server_entry) { } } +ErrorCode Client::ConnectToCvmSubmasters(const std::string& etcd_endpoints) { + ErrorCode err = EtcdHelper::ConnectToEtcdStoreClient(etcd_endpoints); + if (err != ErrorCode::OK) { + LOG(ERROR) << "ConnectToCvmSubmasters: failed to connect etcd: " << err; + return err; + } + + // Discover cluster namespaces that have registered masters by scanning + // the /cvm/ key space for /cvm//masters/ keys. + const std::string cvm_root(std::string(mooncake::cvm::kCvmRootPrefix)); + const std::string cvm_root_end = mooncake::cvm::PrefixEnd(cvm_root); + const std::string masters_suffix("/masters/"); + std::string json; + EtcdRevisionId revision = 0; + err = EtcdHelper::GetRangeAsJson(cvm_root.data(), cvm_root.size(), + cvm_root_end.data(), cvm_root_end.size(), + 0 /* no limit */, json, revision); + if (err != ErrorCode::OK) { + LOG(ERROR) << "ConnectToCvmSubmasters: failed to scan /cvm/ prefix: " + << err; + return err; + } + + std::set namespaces; + { + Json::Value root; + Json::CharReaderBuilder reader; + std::string errors; + std::istringstream stream(json); + if (!Json::parseFromStream(reader, stream, &root, &errors) || + !root.isArray()) { + LOG(ERROR) << "ConnectToCvmSubmasters: failed to parse /cvm/ " + "range JSON: " + << errors; + return ErrorCode::INTERNAL_ERROR; + } + for (const auto& item : root) { + if (!item.isObject() || !item["key"].isString()) { + continue; + } + const std::string key = item["key"].asString(); + if (!key.starts_with(cvm_root)) { + continue; + } + const size_t ns_begin = cvm_root.size(); + const size_t masters_pos = key.find(masters_suffix, ns_begin); + // Key layout: /cvm//masters/. Reject namespaces + // containing '/' (not a valid masters key). + if (masters_pos == std::string::npos || + key.find('/', ns_begin) != masters_pos) { + continue; + } + namespaces.insert(key.substr(ns_begin, masters_pos - ns_begin)); + } + } + if (namespaces.empty()) { + LOG(ERROR) << "ConnectToCvmSubmasters: no /cvm//masters/ " + "registry found in etcd"; + return ErrorCode::UNAVAILABLE_IN_CURRENT_STATUS; + } + if (namespaces.size() > 1) { + std::ostringstream oss; + for (const auto& ns : namespaces) { + oss << " " << ns; + } + LOG(WARNING) << "ConnectToCvmSubmasters: multiple CVM namespaces " + "found (" + << oss.str() << "); using the first one"; + } + + // Try each namespace; within a namespace, prefer primaries ordered by + // registration time (first-registered first), matching the + // CvmController first-come-first-served ranking. Fall back to standbys + // only if no primary is reachable, so the client can still bootstrap. + for (const auto& ns : namespaces) { + std::vector masters; + ViewVersionId version = 0; + err = cvm::EtcdViewStore::LoadAllMasters(ns, masters, version); + if (err != ErrorCode::OK) { + LOG(WARNING) << "ConnectToCvmSubmasters: LoadAllMasters failed " + "for namespace " + << ns << ": " << err; + continue; + } + std::sort(masters.begin(), masters.end(), + [](const cvm::MasterRegistration& a, + const cvm::MasterRegistration& b) { + if (a.registered_at_ms != b.registered_at_ms) { + return a.registered_at_ms < b.registered_at_ms; + } + return a.master_id < b.master_id; + }); + + auto try_connect_group = [&](bool primaries_only) { + for (const auto& reg : masters) { + if (primaries_only && + reg.role != static_cast(cvm::MasterRole::kPrimary)) { + continue; + } + auto connect_err = master_client_.Connect(reg.address); + if (connect_err != ErrorCode::OK) { + LOG(WARNING) << "ConnectToCvmSubmasters: connect to " + << reg.master_id << " (" << reg.address + << ") failed: " << toString(connect_err); + continue; + } + cvm_cluster_namespace_ = ns; + // 收集该 namespace 下所有 primary 地址，用于 client 侧全量 + // mount 与多 submaster 心跳（同一 segment 挂到所有 submaster）。 + { + std::lock_guard lock( + cvm_submaster_addresses_mutex_); + cvm_submaster_addresses_.clear(); + // 去重：etcd 中可能存在指向同一 address 的重复 primary + // 注册（如重启残留 lease），不去重会导致全量 mount 对同一 + // submaster 重复调用、master 日志刷屏。 + std::set seen; + for (const auto& reg2 : masters) { + if (reg2.role == + static_cast( + cvm::MasterRole::kPrimary) && + !reg2.address.empty() && + seen.insert(reg2.address).second) { + cvm_submaster_addresses_.push_back(reg2.address); + } + } + } + { + std::lock_guard lock(leader_switch_mutex_); + current_master_view_.reset(); + } + // Seed cluster_id_ with the CVM namespace so + // TryLoadRoutingOnce skips fsdir-based discovery: GetFsdir + // returns an empty string when the submaster has no storage + // root configured, which would otherwise leave the partition + // routing permanently unloaded. + { + std::lock_guard lock(routing_load_mutex_); + cluster_id_ = ns; + } + last_ping_success_.store(true); + LOG(INFO) << "ConnectToCvmSubmasters: connected to submaster " + << reg.master_id << " (" << reg.address + << ", role=" << reg.role + << ", namespace=" << ns << ")"; + return ErrorCode::OK; + } + return ErrorCode::UNAVAILABLE_IN_CURRENT_STATUS; + }; + + if (try_connect_group(true) == ErrorCode::OK) { + return ErrorCode::OK; + } + if (try_connect_group(false) == ErrorCode::OK) { + return ErrorCode::OK; + } + LOG(WARNING) << "ConnectToCvmSubmasters: no reachable master in " + "namespace " + << ns; + } + + LOG(ERROR) << "ConnectToCvmSubmasters: no reachable submaster in any " + "CVM namespace"; + return ErrorCode::UNAVAILABLE_IN_CURRENT_STATUS; +} + +void Client::LoadPartitionRouting() { + // Routing is only meaningful in HA mode where etcd is available. + if (etcd_connstring_.empty()) { + LOG(INFO) << "LoadPartitionRouting skipped: non-HA mode (no etcd " + "endpoints)"; + return; + } + LOG(INFO) << "LoadPartitionRouting: etcd_endpoints=[" << etcd_connstring_ + << "]"; + + // First attempt (best effort). A failure here must not fail client + // startup; the refresh loop started below keeps retrying until it + // succeeds. + TryLoadRoutingOnce(); + + // Start the periodic refresh/retry loop unconditionally in HA mode, so a + // first-attempt failure (etcd temporarily unavailable, GetFsdir hiccup) is + // retried on the next cycle instead of leaving the client permanently in + // single-master fallback. + if (!routing_refresh_running_.exchange(true)) { + routing_refresh_thread_ = + std::thread([this]() { this->RoutingRefreshThreadMain(); }); + } +} + +void Client::TryLoadRoutingOnce() { + // 串行化 routing-refresh 线程与 SLOT_NOT_OWNED 触发的按需刷新， + // 避免并发读写 cluster_id_ 与路由表。 + std::lock_guard lock(routing_load_mutex_); + // Resolve the cluster namespace (cluster_id) from the fsdir returned by + // master, which is formatted as "/". The result is + // stable, so it is cached in cluster_id_ and reused across retries. + std::string cluster_id = cluster_id_; + if (cluster_id.empty()) { + auto fsdir_response = master_client_.GetFsdir(); + if (!fsdir_response) { + LOG(WARNING) << "TryLoadRoutingOnce: GetFsdir failed, err=" + << toString(fsdir_response.error()) << " (will retry)"; + return; + } + const std::string& fsdir = fsdir_response.value(); + if (fsdir.empty()) { + LOG(WARNING) << "TryLoadRoutingOnce: GetFsdir returned empty fsdir"; + return; + } + LOG(INFO) << "TryLoadRoutingOnce: fsdir=" << fsdir; + + const size_t pos = fsdir.find_last_of('/'); + if (pos == std::string::npos) { + LOG(WARNING) << "TryLoadRoutingOnce: invalid fsdir format (no '/'), " + "fsdir=" + << fsdir; + return; + } + cluster_id = fsdir.substr(pos + 1); + if (cluster_id.empty()) { + LOG(WARNING) << "TryLoadRoutingOnce: empty cluster_id parsed from " + "fsdir=" + << fsdir; + return; + } + cluster_id_ = cluster_id; + LOG(INFO) << "TryLoadRoutingOnce: resolved cluster_id=" << cluster_id; + } + + ErrorCode err = + master_client_.LoadRoutingFromEtcd(etcd_connstring_, cluster_id); + if (err != ErrorCode::OK) { + LOG(WARNING) << "TryLoadRoutingOnce: failed to load routing, " + "cluster_id=" + << cluster_id << " err=" << err; + return; + } + LOG(INFO) << "TryLoadRoutingOnce: routing loaded, cluster_id=" << cluster_id; +} + +void Client::RoutingRefreshThreadMain() { + // Refresh cadence aligned with SlotOwnerHeartbeat (5s) and + // CvmController::SyncLoop (5s). Slot migration is a low-frequency event, + // so a fixed sleep (rather than a fast poll) keeps steady-state etcd load + // low while still picking up changes promptly. + constexpr auto kRefreshInterval = std::chrono::milliseconds(5000); + + while (routing_refresh_running_.load()) { + std::this_thread::sleep_for(kRefreshInterval); + TryLoadRoutingOnce(); + // CVM 多 submaster：同步刷新已发现的 primary 地址，并对新增的 + // submaster 全量 mount，处理运行期 submaster 增删。 + RefreshSubmasterAddresses(); + } +} + ErrorCode Client::SwitchLeader(const ha::MasterView& target_view) { std::lock_guard lock(leader_switch_mutex_); @@ -923,6 +1214,10 @@ std::optional> Client::Create( return std::nullopt; } + // Load the KV partition routing table from etcd (HA mode only). Best + // effort: a routing load failure must not fail client startup. + client->LoadPartitionRouting(); + // Initialize storage backend if storage_root_dir is valid auto config_response = client->master_client_.GetStorageConfig(); if (!config_response) { @@ -1100,6 +1395,19 @@ tl::expected Client::Query( std::chrono::steady_clock::time_point start_time = std::chrono::steady_clock::now(); auto result = master_client_.GetReplicaList(object_key); + // 松耦合重路由：SLOT_NOT_OWNED 表示本地 slot→submaster 视图过期 + // （分区已重平衡）。刷新一次路由快照后重试，submaster 经 etcd 收敛视图。 + if (!result && result.error() == ErrorCode::SLOT_NOT_OWNED && + !etcd_connstring_.empty()) { + LOG(WARNING) << "Query: SLOT_NOT_OWNED for key=" << object_key + << ", refreshing slot routing and retrying once"; + TryLoadRoutingOnce(); + result = master_client_.GetReplicaList(object_key); + if (result) { + LOG(INFO) << "Query: re-route retry succeeded for key=" + << object_key; + } + } if (!result) { return tl::unexpected(result.error()); } @@ -1120,6 +1428,26 @@ std::vector> Client::BatchQuery( std::chrono::steady_clock::now(); auto response = master_client_.BatchGetReplicaList(object_keys, tenant_id); + // 松耦合重路由：任一项返回 SLOT_NOT_OWNED 说明视图过期，刷新一次后 + // 整体重试；读操作幂等，重复查询已成功项无害。 + if (!etcd_connstring_.empty()) { + bool has_slot_not_owned = false; + for (const auto& r : response) { + if (!r && r.error() == ErrorCode::SLOT_NOT_OWNED) { + has_slot_not_owned = true; + break; + } + } + if (has_slot_not_owned) { + LOG(WARNING) << "BatchQuery: SLOT_NOT_OWNED in batch of " + << object_keys.size() + << " keys, refreshing slot routing and retrying once"; + TryLoadRoutingOnce(); + response = + master_client_.BatchGetReplicaList(object_keys, tenant_id); + } + } + // Check if we got the expected number of responses if (response.size() != object_keys.size()) { LOG(ERROR) << "BatchQuery response size mismatch. Expected: " @@ -3134,12 +3462,30 @@ tl::expected Client::MountSegment( tl::expected Client::UnmountSegmentImpl( std::unordered_map>::iterator it) { - auto unmount_result = master_client_.UnmountSegment(it->second.id); - if (!unmount_result) { - ErrorCode err = unmount_result.error(); - LOG(ERROR) << "Failed to unmount segment from master: " - << toString(err); - return tl::unexpected(err); + // 全量 unmount：与全量 mount 对称，向所有 primary submaster 注销 segment + //（单个失败仅告警，不阻断本地 unregister）。单 master/直连走原路径。 + std::vector unmount_targets; + { + std::lock_guard addr_lock(cvm_submaster_addresses_mutex_); + unmount_targets = cvm_submaster_addresses_; + } + if (!unmount_targets.empty()) { + for (const auto& addr : unmount_targets) { + auto r = master_client_.UnmountSegmentTo(addr, it->second.id); + if (!r) { + LOG(WARNING) << "unmount_segment_from_submaster_failed addr=" + << addr << " id=" << it->second.id + << " error=" << r.error(); + } + } + } else { + auto unmount_result = master_client_.UnmountSegment(it->second.id); + if (!unmount_result) { + ErrorCode err = unmount_result.error(); + LOG(ERROR) << "Failed to unmount segment from master: " + << toString(err); + return tl::unexpected(err); + } } int rc = transfer_engine_->unregisterLocalMemory( @@ -3228,12 +3574,40 @@ tl::expected Client::MountSegmentAndGetId( segment.te_endpoint = local_hostname_; } - auto mount_result = master_client_.MountSegment(segment); - if (!mount_result) { - ErrorCode err = mount_result.error(); - LOG(ERROR) << "mount_segment_to_master_failed base=" << buffer - << " size=" << size << ", error=" << err; - return tl::unexpected(err); + // 全量 mount：CVM 多 submaster 模式下，同一 segment 挂到所有 primary + // submaster（任一成功即可），使任何 slot owner 都能本地分配副本； + // 单 master/直连模式下 cvm_submaster_addresses_ 为空，走原 MountSegment。 + std::vector mount_targets; + { + std::lock_guard addr_lock( + cvm_submaster_addresses_mutex_); + mount_targets = cvm_submaster_addresses_; + } + if (!mount_targets.empty()) { + bool any_mounted = false; + for (const auto& addr : mount_targets) { + auto r = master_client_.MountSegmentTo(addr, segment); + if (r) { + any_mounted = true; + } else { + LOG(WARNING) << "mount_segment_to_submaster_failed addr=" + << addr << " id=" << segment.id + << " error=" << r.error(); + } + } + if (!any_mounted) { + LOG(ERROR) << "mount_segment_to_all_submasters_failed base=" + << buffer << " size=" << size; + return tl::unexpected(ErrorCode::RPC_FAIL); + } + } else { + auto mount_result = master_client_.MountSegment(segment); + if (!mount_result) { + ErrorCode err = mount_result.error(); + LOG(ERROR) << "mount_segment_to_master_failed base=" << buffer + << " size=" << size << ", error=" << err; + return tl::unexpected(err); + } } segment_id = segment.id; @@ -3258,13 +3632,35 @@ tl::expected Client::UnmountSegmentById( return UnmountSegmentImpl(segment); } - auto result = - master_client_.GracefulUnmountSegment(segment_id, grace_period_ms); - if (!result) { - ErrorCode err = result.error(); - LOG(ERROR) << "Failed to graceful unmount segment from master: " - << toString(err); - return tl::unexpected(err); + // 全量优雅卸载：与全量 mount/unmount 对称，向所有 primary submaster 发起 + // 优雅卸载。单个失败即返回错误，避免部分 submaster 已进入优雅卸载而其余 + // 未生效导致状态不一致。单 master/直连模式走原路径。 + std::vector graceful_targets; + { + std::lock_guard addr_lock(cvm_submaster_addresses_mutex_); + graceful_targets = cvm_submaster_addresses_; + } + if (!graceful_targets.empty()) { + for (const auto& addr : graceful_targets) { + auto r = master_client_.GracefulUnmountSegmentTo( + addr, segment_id, grace_period_ms); + if (!r) { + LOG(ERROR) << "Failed to graceful unmount segment from " + "submaster addr=" + << addr << " id=" << segment_id + << " error=" << r.error(); + return tl::unexpected(r.error()); + } + } + } else { + auto result = + master_client_.GracefulUnmountSegment(segment_id, grace_period_ms); + if (!result) { + ErrorCode err = result.error(); + LOG(ERROR) << "Failed to graceful unmount segment from master: " + << toString(err); + return tl::unexpected(err); + } } gracefully_unmounting_segments_.emplace(segment->first, segment->second); @@ -3306,17 +3702,47 @@ void Client::OnGracefulUnmountTimer(const UUID& segment_id, int retry_left) { } } - auto status = master_client_.QuerySegmentStatusById(segment_id); + std::vector status_targets; + { + std::lock_guard addr_lock(cvm_submaster_addresses_mutex_); + status_targets = cvm_submaster_addresses_; + } + bool removed = false; - if (!status) { - if (status.error() == ErrorCode::SEGMENT_NOT_FOUND) { + if (!status_targets.empty()) { + // 多 submaster：仅当所有 primary submaster 均确认移除后才视为完成， + // 保证优雅卸载在所有 submaster 对称生效后才释放本地 MR 与回调。 + removed = true; + for (const auto& addr : status_targets) { + auto status = + master_client_.QuerySegmentStatusByIdTo(addr, segment_id); + if (!status) { + if (status.error() == ErrorCode::SEGMENT_NOT_FOUND) { + continue; // 该 submaster 已移除 + } + LOG(WARNING) << "Failed to query graceful unmount segment " + "status from submaster addr=" + << addr << " error=" << status.error(); + removed = false; + } else if (status.value() == SegmentStatus::UNDEFINED) { + continue; // 该 submaster 已移除 + } else { + removed = false; + } + } + } else { + auto status = master_client_.QuerySegmentStatusById(segment_id); + if (!status) { + if (status.error() == ErrorCode::SEGMENT_NOT_FOUND) { + removed = true; + } else { + LOG(WARNING) + << "Failed to query graceful unmount segment status: " + << toString(status.error()); + } + } else if (status.value() == SegmentStatus::UNDEFINED) { removed = true; - } else { - LOG(WARNING) << "Failed to query graceful unmount segment status: " - << toString(status.error()); } - } else if (status.value() == SegmentStatus::UNDEFINED) { - removed = true; } if (removed) { @@ -3505,6 +3931,16 @@ tl::expected Client::PromotionObjectHeartbeat( return {}; } +tl::expected, ErrorCode> +Client::RemoveObjectHeartbeat(const UUID& client_id) { + return master_client_.RemoveObjectHeartbeat(client_id); +} + +tl::expected Client::AckRemoveObjectHeartbeat( + const UUID& client_id, const std::vector& tasks) { + return master_client_.AckRemoveObjectHeartbeat(client_id, tasks); +} + tl::expected Client::PromotionAllocStart( const std::string& key, uint64_t size, @@ -4123,6 +4559,143 @@ void Client::ExecuteTask(const ClientTask& client_task) { } } +void Client::HeartbeatAllSubmasters() { + std::vector targets; + { + std::lock_guard lock(cvm_submaster_addresses_mutex_); + targets = cvm_submaster_addresses_; + } + if (targets.empty()) { + return; // 单 master/直连模式，无多 submaster 心跳 + } + + // 去重：当前 active master 由主循环 master_client_.Ping() 单独 ping， + // 这里跳过，避免同一 submaster 被重复 ping。 + const std::string active_address = master_client_.GetCurrentAddress(); + + for (const auto& addr : targets) { + if (addr == active_address) { + continue; + } + auto ping_result = master_client_.PingTo(addr); + if (!ping_result) { + VLOG(1) << "HeartbeatAllSubmasters: ping submaster failed addr=" + << addr << " error=" << ping_result.error(); + continue; + } + if (ping_result.value().client_status == ClientStatus::NEED_REMOUNT) { + // 该 submaster 侧没有本 client 的 segment（可能被误判过期卸载）， + // 重新 mount 所有本地 segment 到该 submaster，恢复其分配能力。 + LOG(INFO) << "HeartbeatAllSubmasters: submaster " << addr + << " requires remount"; + std::lock_guard lock(mounted_segments_mutex_); + for (const auto& [seg_id, segment] : mounted_segments_) { + auto r = master_client_.MountSegmentTo(addr, segment); + if (!r) { + LOG(WARNING) << "remount_segment_to_submaster_failed addr=" + << addr << " id=" << seg_id + << " error=" << r.error(); + } + } + } + } +} + +void Client::RefreshSubmasterAddresses() { + // 仅在 CVM 多 submaster 模式下有意义（有已知 namespace）。 + if (cvm_cluster_namespace_.empty()) { + return; + } + + std::vector masters; + ViewVersionId version = 0; + ErrorCode err = cvm::EtcdViewStore::LoadAllMasters(cvm_cluster_namespace_, + masters, version); + if (err != ErrorCode::OK) { + LOG(WARNING) << "RefreshSubmasterAddresses: LoadAllMasters failed: " + << err; + return; + } + + std::vector new_addresses; + for (const auto& reg : masters) { + if (reg.role == static_cast(cvm::MasterRole::kPrimary) && + !reg.address.empty()) { + new_addresses.push_back(reg.address); + } + } + std::sort(new_addresses.begin(), new_addresses.end()); + // 去重：与 ConnectToCvmSubmasters 保持一致，避免重复 primary 注册导致 + // 全量 mount 对同一 submaster 重复调用。 + new_addresses.erase( + std::unique(new_addresses.begin(), new_addresses.end()), + new_addresses.end()); + + // 若本轮扫描为空（etcd 抖动或所有 primary 异常），保持现有地址列表不变 + //（sticky），避免短暂异常清空地址列表导致心跳与全量 mount 退化。 + if (new_addresses.empty()) { + LOG(WARNING) << "RefreshSubmasterAddresses: no primary masters found, " + "keeping existing addresses"; + return; + } + + // 对比现有地址，找出新增/移除的 submaster；同时用最新结果替换地址列表。 + std::vector added; + std::vector removed; + { + std::lock_guard lock(cvm_submaster_addresses_mutex_); + for (const auto& addr : new_addresses) { + if (std::find(cvm_submaster_addresses_.begin(), + cvm_submaster_addresses_.end(), + addr) == cvm_submaster_addresses_.end()) { + added.push_back(addr); + } + } + for (const auto& addr : cvm_submaster_addresses_) { + if (std::find(new_addresses.begin(), new_addresses.end(), addr) == + new_addresses.end()) { + removed.push_back(addr); + } + } + cvm_submaster_addresses_ = std::move(new_addresses); + } + + // 对新增的 submaster 全量 mount 所有本地 segment，使其具备分配能力。 + if (!added.empty()) { + std::lock_guard lock(mounted_segments_mutex_); + for (const auto& addr : added) { + LOG(INFO) << "RefreshSubmasterAddresses: new submaster " << addr + << ", mounting all local segments"; + for (const auto& [seg_id, segment] : mounted_segments_) { + auto r = master_client_.MountSegmentTo(addr, segment); + if (!r) { + LOG(WARNING) << "mount_segment_to_new_submaster_failed addr=" + << addr << " id=" << seg_id + << " error=" << r.error(); + } + } + } + } + + // 对被移除的 submaster 全量 unmount，保证 segment 生命周期对称闭合。 + // 单个失败仅告警不阻断（其 lease 最终会过期回收）。 + if (!removed.empty()) { + std::lock_guard lock(mounted_segments_mutex_); + for (const auto& addr : removed) { + LOG(INFO) << "RefreshSubmasterAddresses: submaster removed " << addr + << ", unmounting all local segments"; + for (const auto& [seg_id, segment] : mounted_segments_) { + auto r = master_client_.UnmountSegmentTo(addr, segment.id); + if (!r) { + LOG(WARNING) + << "unmount_segment_from_removed_submaster_failed addr=" + << addr << " id=" << seg_id << " error=" << r.error(); + } + } + } + } +} + void Client::StorageHeartbeatThreadMain() { // How many failed pings before reconnecting via the HA coordinator const int max_ping_fail_count = 3; @@ -4199,6 +4772,10 @@ void Client::StorageHeartbeatThreadMain() { remount_segment_future = std::future(); } + // CVM 多 submaster 心跳：向所有 primary submaster 定向 ping 保活， + // 避免各 submaster 因收不到本 client 的 ping 而误判过期并卸载 segment。 + HeartbeatAllSubmasters(); + // Ping master auto ping_result = master_client_.Ping(); if (ping_result) { @@ -4294,6 +4871,20 @@ void Client::StorageHeartbeatThreadMain() { LOG(INFO) << "Reconnected to master " << next_view.leader_address; ping_fail_count = 0; + } else if (!cvm_cluster_namespace_.empty()) { + // CVM multi-submaster mode: the connected submaster died or + // became unreachable. Re-discover live submasters from the + // /cvm/ registry and reconnect to another one. + LOG(ERROR) << "Failed to ping master for " << ping_fail_count + << " times (CVM submaster mode); re-discovering " + "submasters from etcd"; + auto err = ConnectToCvmSubmasters(etcd_connstring_); + if (err != ErrorCode::OK) { + std::this_thread::sleep_for( + std::chrono::milliseconds(fail_ping_interval_ms)); + continue; + } + ping_fail_count = 0; } else { const std::string current_master_address = direct_master_address_; LOG(ERROR) << "Failed to ping master for " << ping_fail_count diff --git a/mooncake-store/src/cvm/cvm_controller.cpp b/mooncake-store/src/cvm/cvm_controller.cpp new file mode 100644 index 0000000000..4245a68be9 --- /dev/null +++ b/mooncake-store/src/cvm/cvm_controller.cpp @@ -0,0 +1,588 @@ +#include "cvm/cvm_controller.h" + +#include +#include +#include +#include +#include +#include + +#include + +#include "cvm/cvm_http_server.h" +#include "cvm/cvm_keys.h" +#include "cvm/cvm_service_delegate.h" +#include "cvm/etcd_view_store.h" +#include "cvm/slot_hash.h" +#include "etcd_helper.h" + +namespace mooncake { +namespace cvm { + +namespace { + +constexpr int kWatchEventBroken = 2; +constexpr int kWatchStopTimeoutMs = 1000; +constexpr int kKeepAliveReadyTimeoutMs = 1000; + +} // namespace + +CvmController::CvmController(Config config) : config_(std::move(config)) { + current_role_.store(config_.role); +} + +CvmController::~CvmController() { Stop(); } + +void CvmController::SetDelegate(CvmServiceDelegate* delegate) { + delegate_ = delegate; +} + +ErrorCode CvmController::Start() { + if (running_.load()) { + return ErrorCode::OK; + } + + LOG(INFO) << "CvmController::Start begin: cluster_namespace=" + << config_.cluster_namespace << ", master_id=" + << config_.master_id << ", address=" << config_.address + << ", role=" << static_cast(config_.role) + << ", registration_lease_ttl_sec=" + << config_.registration_lease_ttl_sec << ", http_host=" + << config_.http_host << ", http_port=" << config_.http_port + << ", sync_interval_ms=" << config_.sync_interval.count(); + + // NOTE: the etcd client is a process-global singleton (EtcdHelper); the + // embedding master is responsible for connecting it before Start(). + ErrorCode err = EtcdHelper::GrantLease(config_.registration_lease_ttl_sec, + lease_id_); + if (err != ErrorCode::OK) { + LOG(ERROR) << "CvmController::Start GrantLease failed: err=" << err + << ", cluster_namespace=" << config_.cluster_namespace + << ", master_id=" << config_.master_id + << ", registration_lease_ttl_sec=" + << config_.registration_lease_ttl_sec + << " (etcd client may not be connected yet)"; + return err; + } + LOG(INFO) << "CvmController::Start GrantLease ok: lease_id=" << lease_id_; + + MasterRegistration reg; + reg.master_id = config_.master_id; + reg.address = config_.address; + reg.role = static_cast(config_.role); + reg.registered_at_ms = + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + LOG(INFO) << "CvmController::Start registering master: master_id=" + << reg.master_id << ", address=" << reg.address << ", role=" + << reg.role << ", cluster_namespace=" << config_.cluster_namespace + << ", lease_id=" << lease_id_; + err = EtcdViewStore::RegisterMaster(config_.cluster_namespace, reg, + lease_id_); + if (err != ErrorCode::OK) { + LOG(ERROR) << "CvmController::Start RegisterMaster failed: err=" << err + << ", master_id=" << reg.master_id << ", address=" + << reg.address << ", role=" << reg.role + << ", cluster_namespace=" << config_.cluster_namespace + << ", lease_id=" << lease_id_; + (void)EtcdHelper::RevokeLease(lease_id_); + return err; + } + LOG(INFO) << "CvmController::Start RegisterMaster ok: master_id=" + << reg.master_id; + + err = RefreshKvView(); + if (err != ErrorCode::OK) { + LOG(WARNING) << "CvmController initial RefreshKvView failed: " << err + << ", cluster_namespace=" << config_.cluster_namespace; + } else { + LOG(INFO) << "CvmController initial RefreshKvView ok"; + } + + watch_state_ = std::make_unique(); + masters_watch_state_ = std::make_unique(); + running_.store(true); + + keepalive_thread_ = std::thread([this]() { KeepaliveLoop(); }); + watch_thread_ = std::thread([this]() { WatchLoop(); }); + masters_watch_thread_ = std::thread([this]() { MastersWatchLoop(); }); + sync_thread_ = std::thread([this]() { SyncLoop(); }); + membership_thread_ = std::thread([this]() { MembershipLoop(); }); + LOG(INFO) << "CvmController::Start started " + "keepalive/watch/masters_watch/sync/membership threads"; + + if (config_.http_port != 0) { + CvmHttpServer::Config http_cfg; + http_cfg.host = config_.http_host; + http_cfg.port = config_.http_port; + http_cfg.cluster_namespace = config_.cluster_namespace; + LOG(INFO) << "CvmController::Start starting CvmHttpServer: host=" + << http_cfg.host << ", port=" << http_cfg.port; + http_server_ = std::make_unique(http_cfg); + err = http_server_->Start(); + if (err != ErrorCode::OK) { + LOG(ERROR) << "CvmController start http server failed: " << err + << ", host=" << http_cfg.host << ", port=" + << http_cfg.port; + http_server_.reset(); + } else { + LOG(INFO) << "CvmController CvmHttpServer started"; + } + } + + LOG(INFO) << "CvmController::Start ok"; + return ErrorCode::OK; +} + +void CvmController::Stop() { + if (!running_.load() && !watch_thread_.joinable() && + !masters_watch_thread_.joinable() && !keepalive_thread_.joinable() && + !sync_thread_.joinable() && !membership_thread_.joinable()) { + return; + } + + running_.store(false); + + CancelWatchAndWait(); + + if (watch_state_) { + std::lock_guard lock(watch_state_->mutex); + watch_state_->cv.notify_all(); + } + if (masters_watch_state_) { + std::lock_guard lock(masters_watch_state_->mutex); + masters_watch_state_->cv.notify_all(); + } + + if (keepalive_thread_.joinable()) { + (void)EtcdHelper::CancelKeepAlive(lease_id_); + keepalive_thread_.join(); + } + if (watch_thread_.joinable()) { + watch_thread_.join(); + } + if (masters_watch_thread_.joinable()) { + masters_watch_thread_.join(); + } + if (sync_thread_.joinable()) { + sync_cv_.notify_all(); + sync_thread_.join(); + } + if (membership_thread_.joinable()) { + membership_cv_.notify_all(); + membership_thread_.join(); + } + + if (http_server_) { + http_server_->Stop(); + http_server_.reset(); + } + + (void)EtcdHelper::RevokeLease(lease_id_); + watch_state_.reset(); + masters_watch_state_.reset(); +} + +bool CvmController::OwnsSlot(uint16_t slot) const { + SharedMutexLocker locker(&view_mutex_, shared_lock); + auto it = kv_view_.find(slot); + if (it == kv_view_.end()) { + return false; + } + return it->second.primary_master_id == config_.master_id && + it->second.state == static_cast(SlotState::kStable); +} + +std::vector CvmController::GetKvView() const { + SharedMutexLocker locker(&view_mutex_, shared_lock); + std::vector out; + out.reserve(kv_view_.size()); + for (const auto& entry : kv_view_) { + out.push_back(entry.second); + } + return out; +} + +ViewVersionId CvmController::GetKvViewVersion() const { + SharedMutexLocker locker(&view_mutex_, shared_lock); + return kv_view_version_; +} + +ErrorCode CvmController::SyncOnce() { + ViewVersionId version = 0; + ErrorCode err = EtcdViewStore::BuildAndSaveKvViewSnapshot( + config_.cluster_namespace, version); + if (err != ErrorCode::OK) { + LOG(ERROR) << "CvmController build kv view snapshot failed: " << err; + return err; + } + + err = EtcdViewStore::BuildAndSaveSegmentViewSnapshot( + config_.cluster_namespace, version); + if (err != ErrorCode::OK) { + // segment 视图为预留，失败不阻断 kv 视图路径同步。 + LOG(WARNING) << "CvmController build segment view snapshot failed: " + << err; + } + + PushViewPaths(); + return ErrorCode::OK; +} + +void CvmController::PushViewPaths() { + const std::string kv_key = KvViewSnapshotKey(config_.cluster_namespace); + const std::string segment_key = + SegmentViewSnapshotKey(config_.cluster_namespace); + + view_paths_["kv_view"] = kv_key; + view_paths_["segment_view"] = segment_key; + + if (http_server_) { + http_server_->SetKvViewSnapshotKey(kv_key); + http_server_->SetSegmentViewSnapshotKey(segment_key); + } + LOG(INFO) << "CvmController pushed view paths: kv_view=" << kv_key + << " segment_view=" << segment_key; +} + +ErrorCode CvmController::RefreshKvView() { + std::vector owners; + ViewVersionId version = 0; + ErrorCode err = + EtcdViewStore::LoadAllSlotOwners(config_.cluster_namespace, owners, + version); + if (err != ErrorCode::OK) { + return err; + } + + std::unordered_map next; + next.reserve(owners.size()); + for (auto& owner : owners) { + next.emplace(owner.slot, std::move(owner)); + } + + { + SharedMutexLocker locker(&view_mutex_); + kv_view_ = std::move(next); + kv_view_version_ = version; + } + return ErrorCode::OK; +} + +void CvmController::CancelWatchAndWait() { + if (watch_armed_.exchange(false)) { + (void)EtcdViewStore::CancelWatchKvView(config_.cluster_namespace); + (void)EtcdViewStore::WaitWatchKvViewStopped(config_.cluster_namespace, + kWatchStopTimeoutMs); + } + if (masters_watch_armed_.exchange(false)) { + (void)EtcdViewStore::CancelWatchMasters(config_.cluster_namespace); + (void)EtcdViewStore::WaitWatchMastersStopped(config_.cluster_namespace, + kWatchStopTimeoutMs); + } +} + +void CvmController::WatchLoop() { + while (running_.load()) { + ViewVersionId version_before = 0; + { + SharedMutexLocker locker(&view_mutex_, shared_lock); + version_before = kv_view_version_; + } + + ErrorCode err = RefreshKvView(); + if (err != ErrorCode::OK) { + LOG(WARNING) << "CvmController WatchLoop refresh failed: " << err; + } + + ViewVersionId version_after = 0; + { + SharedMutexLocker locker(&view_mutex_, shared_lock); + version_after = kv_view_version_; + } + // slot 所有权视图变化：通知 delegate，让 standby 即使角色不变也能 + // 重绑回放源（技术债 2 修复）。 + if (version_after != version_before && delegate_) { + delegate_->OnKvViewChanged(); + } + + if (!running_.load()) { + break; + } + + CancelWatchAndWait(); + + { + std::lock_guard lock(watch_state_->mutex); + watch_state_->dirty = false; + watch_state_->broken = false; + } + + ViewVersionId start_version = 0; + { + SharedMutexLocker locker(&view_mutex_, shared_lock); + start_version = kv_view_version_; + } + + err = EtcdViewStore::WatchKvView(config_.cluster_namespace, + start_version, watch_state_.get(), + &CvmController::WatchCallback); + if (err != ErrorCode::OK) { + LOG(WARNING) << "CvmController arm watch failed: " << err; + std::this_thread::sleep_for(std::chrono::seconds(1)); + continue; + } + watch_armed_.store(true); + + std::unique_lock lock(watch_state_->mutex); + watch_state_->cv.wait(lock, [this] { + return watch_state_->dirty || watch_state_->broken || + !running_.load(); + }); + } +} + +void CvmController::MastersWatchLoop() { + while (running_.load()) { + { + std::lock_guard lock(masters_watch_state_->mutex); + masters_watch_state_->dirty = false; + masters_watch_state_->broken = false; + } + + // start_revision=0 → 从当前开始 watch，只关注未来的成员增删（lease + // 过期删除 / 新节点注册）。 + ErrorCode err = EtcdViewStore::WatchMasters( + config_.cluster_namespace, /*start_revision=*/0, + masters_watch_state_.get(), &CvmController::WatchCallback); + if (err != ErrorCode::OK) { + LOG(WARNING) << "CvmController arm masters watch failed: " << err; + std::this_thread::sleep_for(std::chrono::seconds(1)); + continue; + } + masters_watch_armed_.store(true); + + std::unique_lock lock(masters_watch_state_->mutex); + masters_watch_state_->cv.wait(lock, [this] { + return masters_watch_state_->dirty || + masters_watch_state_->broken || !running_.load(); + }); + + // 成员增删（lease 过期）→ 立即重算角色，并唤醒 sync 线程立即重算 + // slot 快照，使 failover 不再等待轮询周期。 + ReconcileRole(); + sync_cv_.notify_all(); + } +} + +void CvmController::KeepaliveLoop() { + (void)EtcdHelper::WaitKeepAliveReady(lease_id_, kKeepAliveReadyTimeoutMs); + ErrorCode rc = EtcdHelper::KeepAlive(lease_id_); + if (rc == ErrorCode::ETCD_OPERATION_ERROR) { + LOG(WARNING) << "CvmController keepalive error: " << rc; + } +} + +void CvmController::SyncLoop() { + while (running_.load()) { + (void)SyncOnce(); + + std::unique_lock lock(sync_mutex_); + sync_cv_.wait_for(lock, config_.sync_interval, + [this] { return !running_.load(); }); + } +} + +bool CvmController::LoadRankedMembers( + std::vector& out) { + std::vector masters; + ViewVersionId version = 0; + ErrorCode err = EtcdViewStore::LoadAllMasters(config_.cluster_namespace, + masters, version); + if (err != ErrorCode::OK) { + LOG(WARNING) << "CvmController load masters failed: " << err + << ", master_id=" << config_.master_id; + return false; + } + + // 收集所有存活 master（含本机）作为候选，按「先到先得」排序。 + out.clear(); + out.reserve(masters.size()); + for (const auto& m : masters) { + if (!m.master_id.empty()) { + out.push_back(m); + } + } + + std::sort(out.begin(), out.end(), + [](const MasterRegistration& a, const MasterRegistration& b) { + if (a.registered_at_ms != b.registered_at_ms) { + return a.registered_at_ms < b.registered_at_ms; + } + return a.master_id < b.master_id; + }); + return true; +} + +MasterRole CvmController::ComputeDesiredRole() { + std::vector members; + if (!LoadRankedMembers(members)) { + return current_role_.load(); + } + + for (size_t i = 0; i < members.size(); ++i) { + if (members[i].master_id == config_.master_id) { + return i < config_.submaster_count ? MasterRole::kPrimary + : MasterRole::kStandby; + } + } + + // 本机不在成员集（未注册/异常）：保守保持当前角色。 + return current_role_.load(); +} + +std::string CvmController::GetPrimaryAddress() { + std::vector members; + if (!LoadRankedMembers(members) || members.empty()) { + return ""; + } + // 排名第一的成员是集群中最早的 primary。若本机就是它，则无需（也不能） + // 把自己当作回放源，返回空字符串让调用方保持纯 standby。 + if (members.front().master_id == config_.master_id) { + return ""; + } + return members.front().address; +} + +std::vector CvmController::GetPrimaryPeers() { + std::vector members; + if (!LoadRankedMembers(members) || members.empty()) { + return {}; + } + std::vector peers; + peers.reserve(std::min(members.size(), config_.submaster_count)); + for (size_t i = 0; i < members.size() && i < config_.submaster_count; ++i) { + if (members[i].master_id == config_.master_id) { + continue; // 本机不作为自己的回放源。 + } + peers.push_back(members[i]); + } + return peers; +} + +std::vector CvmController::GetBindingSources() { + std::vector members; + if (!LoadRankedMembers(members) || members.empty()) { + return {}; + } + + const size_t primary_count = + std::min(members.size(), config_.submaster_count); + + // 本机在「先到先得」排序列表中的位置。 + size_t my_index = members.size(); + for (size_t i = 0; i < members.size(); ++i) { + if (members[i].master_id == config_.master_id) { + my_index = i; + break; + } + } + // 未注册 / 本机已是 primary：不作为 standby 回放。 + if (my_index == members.size() || my_index < primary_count) { + return {}; + } + + const size_t standby_count = members.size() - primary_count; + if (standby_count == 0) { + return {}; + } + const size_t standby_rank = my_index - primary_count; + + // 本 standby 负责的 slot 区间 [start, end)，与其它 standby 均分 16384。 + const uint16_t start = static_cast(standby_rank * kSlotCount / + standby_count); + const uint16_t end = static_cast((standby_rank + 1) * kSlotCount / + standby_count); + + // 求出负责区间内每个 slot 的 primary owner（仅 kStable、非本机）。 + std::set owner_ids; + { + SharedMutexLocker locker(&view_mutex_, shared_lock); + for (uint16_t slot = start; slot < end; ++slot) { + auto it = kv_view_.find(slot); + if (it == kv_view_.end()) { + continue; + } + if (it->second.state != static_cast(SlotState::kStable)) { + continue; + } + const std::string& owner = it->second.primary_master_id; + if (!owner.empty() && owner != config_.master_id) { + owner_ids.insert(owner); + } + } + } + + // 映射 owner master_id -> MasterRegistration（含 address）。 + std::vector sources; + sources.reserve(owner_ids.size()); + for (const auto& m : members) { + if (owner_ids.count(m.master_id)) { + sources.push_back(m); + } + } + return sources; +} + +void CvmController::ReconcileRole() { + const MasterRole desired = ComputeDesiredRole(); + MasterRole current = current_role_.load(); + if (desired == current) { + return; + } + // CAS：membership_thread_ 与 masters_watch_thread_ 并发调用时，仅一个 + // 线程执行角色迁移与通知，避免重复 OnRoleChanged。 + if (!current_role_.compare_exchange_strong(current, desired)) { + return; + } + LOG(INFO) << "CvmController role decision: master_id=" + << config_.master_id << ", current=" + << static_cast(current) + << ", desired=" << static_cast(desired) + << ", submaster_count=" << config_.submaster_count; + if (delegate_) { + delegate_->OnRoleChanged(desired); + } +} + +void CvmController::MembershipLoop() { + while (running_.load()) { + ReconcileRole(); + + std::unique_lock lock(membership_mutex_); + membership_cv_.wait_for(lock, config_.sync_interval, + [this] { return !running_.load(); }); + } +} + +void CvmController::WatchCallback(void* ctx, const char* /*key*/, + size_t /*key_size*/, const char* /*value*/, + size_t /*value_size*/, int event_type, + int64_t /*mod_revision*/) { + auto* state = static_cast(ctx); + if (state == nullptr) { + return; + } + { + std::lock_guard lock(state->mutex); + state->dirty = true; + if (event_type == kWatchEventBroken) { + state->broken = true; + } + } + state->cv.notify_all(); +} + +} // namespace cvm +} // namespace mooncake diff --git a/mooncake-store/src/cvm/cvm_http_server.cpp b/mooncake-store/src/cvm/cvm_http_server.cpp new file mode 100644 index 0000000000..11976cee9e --- /dev/null +++ b/mooncake-store/src/cvm/cvm_http_server.cpp @@ -0,0 +1,133 @@ +#include "cvm/cvm_http_server.h" + +#include + +#include + +#include "cvm/cvm_keys.h" +#include "etcd_helper.h" + +namespace mooncake { +namespace cvm { + +CvmHttpServer::CvmHttpServer(Config config) + : config_(std::move(config)), + server_(std::make_unique(4, config_.port)), + kv_view_snapshot_key_(KvViewSnapshotKey(config_.cluster_namespace)), + segment_view_snapshot_key_( + SegmentViewSnapshotKey(config_.cluster_namespace)) { + InitRoutes(); +} + +CvmHttpServer::~CvmHttpServer() { Stop(); } + +void CvmHttpServer::InitRoutes() { + using namespace coro_http; + + server_->set_http_handler( + "/kv_view", [this](coro_http_request& req, coro_http_response& resp) { + (void)req; + std::string json = GetKvViewJson(); + if (json.empty()) { + resp.set_status_and_content(status_type::not_found, + "kv view snapshot not found"); + return; + } + resp.add_header("Content-Type", "application/json"); + resp.set_status_and_content(status_type::ok, json); + }); + + server_->set_http_handler( + "/segment_view", + [this](coro_http_request& req, coro_http_response& resp) { + (void)req; + std::string json = GetSegmentViewJson(); + if (json.empty()) { + resp.set_status_and_content( + status_type::not_found, "segment view snapshot not found"); + return; + } + resp.add_header("Content-Type", "application/json"); + resp.set_status_and_content(status_type::ok, json); + }); + + server_->set_http_handler( + "/health", [](coro_http_request& req, coro_http_response& resp) { + (void)req; + resp.set_status_and_content(status_type::ok, "OK"); + }); +} + +ErrorCode CvmHttpServer::Start() { + if (running_.load()) { + return ErrorCode::OK; + } + + // async_start() binds synchronously and returns a future that is already + // resolved (hasResult()) when the bind failed. Mirrors + // HttpMetadataServer::start(). + auto ec = server_->async_start(); + if (ec.hasResult()) { + LOG(ERROR) << "CvmHttpServer failed to start on " << config_.host << ":" + << config_.port; + return ErrorCode::RPC_FAIL; + } + running_.store(true); + LOG(INFO) << "CvmHttpServer started on " << config_.host << ":" + << config_.port; + return ErrorCode::OK; +} + +void CvmHttpServer::Stop() { + if (!running_.exchange(false)) { + return; + } + server_->stop(); + LOG(INFO) << "CvmHttpServer stopped"; +} + +void CvmHttpServer::SetKvViewSnapshotKey(const std::string& key) { + std::lock_guard lock(path_mutex_); + kv_view_snapshot_key_ = key; +} + +void CvmHttpServer::SetSegmentViewSnapshotKey(const std::string& key) { + std::lock_guard lock(path_mutex_); + segment_view_snapshot_key_ = key; +} + +std::string CvmHttpServer::GetKvViewJson() const { + std::string key; + { + std::lock_guard lock(path_mutex_); + key = kv_view_snapshot_key_; + } + return ReadSnapshot(key); +} + +std::string CvmHttpServer::GetSegmentViewJson() const { + std::string key; + { + std::lock_guard lock(path_mutex_); + key = segment_view_snapshot_key_; + } + return ReadSnapshot(key); +} + +std::string CvmHttpServer::ReadSnapshot(const std::string& key) const { + if (key.empty()) { + return ""; + } + std::string value; + EtcdRevisionId revision = 0; + ErrorCode err = EtcdHelper::Get(key.data(), key.size(), value, revision); + if (err != ErrorCode::OK) { + LOG(WARNING) << "CvmHttpServer read snapshot failed: " << key + << " err=" << err; + return ""; + } + return value; +} + +} // namespace cvm +} // namespace mooncake diff --git a/mooncake-store/src/cvm/etcd_view_store.cpp b/mooncake-store/src/cvm/etcd_view_store.cpp new file mode 100644 index 0000000000..3e547de2f6 --- /dev/null +++ b/mooncake-store/src/cvm/etcd_view_store.cpp @@ -0,0 +1,636 @@ +#include "cvm/etcd_view_store.h" + +#include +#include +#include +#include +#include +#include +#include + +#include + +#if __has_include() +#include +#else +#include +#endif + +#include "cvm/cvm_keys.h" +#include "etcd_helper.h" +#include "ylt/struct_json/json_reader.h" +#include "ylt/struct_json/json_writer.h" + +namespace mooncake { +namespace cvm { + +namespace { + +// Parses the JSON array returned by EtcdHelper::GetRangeAsJson, which has the +// form [{"key":"...","value":"..."}, ...]. +ErrorCode ParseRangeJson(const std::string& json, + std::vector>& kvs) { + Json::Value root; + Json::CharReaderBuilder reader; + std::string errors; + std::istringstream stream(json); + if (!Json::parseFromStream(reader, stream, &root, &errors) || + !root.isArray()) { + LOG(ERROR) << "Failed to parse etcd range JSON: " << errors; + return ErrorCode::INTERNAL_ERROR; + } + + kvs.clear(); + kvs.reserve(root.size()); + for (const auto& item : root) { + if (!item.isObject() || !item["key"].isString() || + !item["value"].isString()) { + return ErrorCode::INTERNAL_ERROR; + } + kvs.emplace_back(item["key"].asString(), item["value"].asString()); + } + return ErrorCode::OK; +} + +} // namespace + +// ---- JSON serialization ---- + +ErrorCode EtcdViewStore::SerializeSlotOwner(const SlotOwner& owner, + std::string& out) { + try { + struct_json::to_json(owner, out); + } catch (const std::exception& e) { + LOG(ERROR) << "SerializeSlotOwner failed: " << e.what(); + return ErrorCode::SERIALIZE_FAIL; + } + return ErrorCode::OK; +} + +ErrorCode EtcdViewStore::DeserializeSlotOwner(const std::string& in, + SlotOwner& out) { + try { + struct_json::from_json(out, in); + } catch (const std::exception& e) { + LOG(ERROR) << "DeserializeSlotOwner failed: " << e.what(); + return ErrorCode::DESERIALIZE_FAIL; + } + return ErrorCode::OK; +} + +ErrorCode EtcdViewStore::SerializeSegmentOwner(const SegmentOwner& owner, + std::string& out) { + try { + struct_json::to_json(owner, out); + } catch (const std::exception& e) { + LOG(ERROR) << "SerializeSegmentOwner failed: " << e.what(); + return ErrorCode::SERIALIZE_FAIL; + } + return ErrorCode::OK; +} + +ErrorCode EtcdViewStore::DeserializeSegmentOwner(const std::string& in, + SegmentOwner& out) { + try { + struct_json::from_json(out, in); + } catch (const std::exception& e) { + LOG(ERROR) << "DeserializeSegmentOwner failed: " << e.what(); + return ErrorCode::DESERIALIZE_FAIL; + } + return ErrorCode::OK; +} + +ErrorCode EtcdViewStore::SerializeMasterRegistration(const MasterRegistration& reg, + std::string& out) { + try { + struct_json::to_json(reg, out); + } catch (const std::exception& e) { + LOG(ERROR) << "SerializeMasterRegistration failed: " << e.what(); + return ErrorCode::SERIALIZE_FAIL; + } + return ErrorCode::OK; +} + +ErrorCode EtcdViewStore::DeserializeMasterRegistration(const std::string& in, + MasterRegistration& out) { + try { + struct_json::from_json(out, in); + } catch (const std::exception& e) { + LOG(ERROR) << "DeserializeMasterRegistration failed: " << e.what(); + return ErrorCode::DESERIALIZE_FAIL; + } + return ErrorCode::OK; +} + +ErrorCode EtcdViewStore::SerializeKvViewSnapshot(const KvViewSnapshot& snapshot, + std::string& out) { + try { + struct_json::to_json(snapshot, out); + } catch (const std::exception& e) { + LOG(ERROR) << "SerializeKvViewSnapshot failed: " << e.what(); + return ErrorCode::SERIALIZE_FAIL; + } + return ErrorCode::OK; +} + +ErrorCode EtcdViewStore::DeserializeKvViewSnapshot(const std::string& in, + KvViewSnapshot& out) { + try { + struct_json::from_json(out, in); + } catch (const std::exception& e) { + LOG(ERROR) << "DeserializeKvViewSnapshot failed: " << e.what(); + return ErrorCode::DESERIALIZE_FAIL; + } + return ErrorCode::OK; +} + +ErrorCode EtcdViewStore::SerializeSegmentViewSnapshot( + const SegmentViewSnapshot& snapshot, std::string& out) { + try { + struct_json::to_json(snapshot, out); + } catch (const std::exception& e) { + LOG(ERROR) << "SerializeSegmentViewSnapshot failed: " << e.what(); + return ErrorCode::SERIALIZE_FAIL; + } + return ErrorCode::OK; +} + +ErrorCode EtcdViewStore::DeserializeSegmentViewSnapshot(const std::string& in, + SegmentViewSnapshot& out) { + try { + struct_json::from_json(out, in); + } catch (const std::exception& e) { + LOG(ERROR) << "DeserializeSegmentViewSnapshot failed: " << e.what(); + return ErrorCode::DESERIALIZE_FAIL; + } + return ErrorCode::OK; +} + +// ---- KV view ---- + +ErrorCode EtcdViewStore::LoadSlotOwner(const std::string& cluster_namespace, + uint16_t slot, SlotOwner& out, + ViewVersionId& version) { + const std::string key = SlotOwnerKey(cluster_namespace, slot); + std::string value; + ErrorCode err = EtcdHelper::Get(key.data(), key.size(), value, version); + if (err != ErrorCode::OK) { + return err; + } + return DeserializeSlotOwner(value, out); +} + +ErrorCode EtcdViewStore::SaveSlotOwner(const std::string& cluster_namespace, + const SlotOwner& owner) { + const std::string key = SlotOwnerKey(cluster_namespace, owner.slot); + std::string value; + ErrorCode err = SerializeSlotOwner(owner, value); + if (err != ErrorCode::OK) { + return err; + } + return EtcdHelper::Put(key.data(), key.size(), value.data(), value.size()); +} + +ErrorCode EtcdViewStore::SaveSlotOwnerWithLease( + const std::string& cluster_namespace, const SlotOwner& owner, + EtcdLeaseId lease_id) { + const std::string key = SlotOwnerKey(cluster_namespace, owner.slot); + std::string value; + ErrorCode err = SerializeSlotOwner(owner, value); + if (err != ErrorCode::OK) { + return err; + } + return EtcdHelper::PutWithLease(key.data(), key.size(), value.data(), + value.size(), lease_id); +} + +ErrorCode EtcdViewStore::SaveSlotOwnersWithLease( + const std::string& cluster_namespace, + const std::vector& owners, EtcdLeaseId lease_id) { + // etcd 默认单事务操作数上限为 128，因此按 128 个/批分块，每批一次事务写。 + constexpr size_t kTxnOpLimit = 128; + for (size_t i = 0; i < owners.size(); i += kTxnOpLimit) { + const size_t n = std::min(kTxnOpLimit, owners.size() - i); + std::vector keys; + std::vector values; + keys.reserve(n); + values.reserve(n); + bool serialize_ok = true; + for (size_t j = 0; j < n; ++j) { + const SlotOwner& owner = owners[i + j]; + keys.push_back(SlotOwnerKey(cluster_namespace, owner.slot)); + std::string value; + if (SerializeSlotOwner(owner, value) != ErrorCode::OK) { + serialize_ok = false; + break; + } + values.push_back(std::move(value)); + } + if (!serialize_ok) { + return ErrorCode::SERIALIZE_FAIL; + } + ErrorCode err = EtcdHelper::BatchPutWithLease(keys, values, lease_id); + if (err == ErrorCode::OK) { + continue; + } + // 某批事务失败：退化为逐条写，避免整批丢弃。仍失败则上抛。 + LOG(WARNING) << "SaveSlotOwnersWithLease batch of " << n + << " failed: " << err << ", falling back per-slot"; + for (size_t j = 0; j < n; ++j) { + err = EtcdHelper::PutWithLease(keys[j].data(), keys[j].size(), + values[j].data(), values[j].size(), + lease_id); + if (err != ErrorCode::OK) { + LOG(ERROR) << "SaveSlotOwnersWithLease per-slot put failed for " + << keys[j] << ": " << err; + return err; + } + } + } + return ErrorCode::OK; +} + +ErrorCode EtcdViewStore::DeleteSlotOwner(const std::string& cluster_namespace, + uint16_t slot) { + const std::string key = SlotOwnerKey(cluster_namespace, slot); + const std::string end = PrefixEnd(key); + return EtcdHelper::DeleteRange(key.data(), key.size(), end.data(), + end.size()); +} + +ErrorCode EtcdViewStore::DeleteSlotOwnerIfOwnedBy( + const std::string& cluster_namespace, uint16_t slot, + const std::string& master_id) { + SlotOwner owner; + ViewVersionId version = 0; + ErrorCode err = LoadSlotOwner(cluster_namespace, slot, owner, version); + if (err == ErrorCode::ETCD_KEY_NOT_EXIST) { + return ErrorCode::OK; // already gone; nothing to clean up + } + if (err != ErrorCode::OK) { + return err; + } + if (owner.primary_master_id != master_id) { + return ErrorCode::OK; // now owned by another master; leave it + } + return DeleteSlotOwner(cluster_namespace, slot); +} + +ErrorCode EtcdViewStore::LoadAllSlotOwners(const std::string& cluster_namespace, + std::vector& out, + ViewVersionId& version) { + out.clear(); + const std::string prefix = KvViewPrefix(cluster_namespace); + const std::string end = PrefixEnd(prefix); + std::string json; + ErrorCode err = EtcdHelper::GetRangeAsJson(prefix.data(), prefix.size(), + end.data(), end.size(), + /*limit=*/0, json, version); + if (err != ErrorCode::OK) { + return err; + } + + std::vector> kvs; + err = ParseRangeJson(json, kvs); + if (err != ErrorCode::OK) { + return err; + } + + out.reserve(kvs.size()); + for (const auto& kv : kvs) { + SlotOwner owner; + err = DeserializeSlotOwner(kv.second, owner); + if (err != ErrorCode::OK) { + return err; + } + out.push_back(std::move(owner)); + } + return ErrorCode::OK; +} + +// ---- Segment view (reserved) ---- + +ErrorCode EtcdViewStore::LoadSegmentOwner(const std::string& cluster_namespace, + const std::string& segment_id, + SegmentOwner& out, + ViewVersionId& version) { + const std::string key = SegmentOwnerKey(cluster_namespace, segment_id); + std::string value; + ErrorCode err = EtcdHelper::Get(key.data(), key.size(), value, version); + if (err != ErrorCode::OK) { + return err; + } + return DeserializeSegmentOwner(value, out); +} + +ErrorCode EtcdViewStore::SaveSegmentOwner(const std::string& cluster_namespace, + const SegmentOwner& owner) { + const std::string key = + SegmentOwnerKey(cluster_namespace, owner.segment_id); + std::string value; + ErrorCode err = SerializeSegmentOwner(owner, value); + if (err != ErrorCode::OK) { + return err; + } + return EtcdHelper::Put(key.data(), key.size(), value.data(), value.size()); +} + +ErrorCode EtcdViewStore::SaveSegmentOwnerWithLease( + const std::string& cluster_namespace, const SegmentOwner& owner, + EtcdLeaseId lease_id) { + const std::string key = + SegmentOwnerKey(cluster_namespace, owner.segment_id); + std::string value; + ErrorCode err = SerializeSegmentOwner(owner, value); + if (err != ErrorCode::OK) { + return err; + } + return EtcdHelper::PutWithLease(key.data(), key.size(), value.data(), + value.size(), lease_id); +} + +ErrorCode EtcdViewStore::DeleteSegmentOwner( + const std::string& cluster_namespace, const std::string& segment_id) { + const std::string key = SegmentOwnerKey(cluster_namespace, segment_id); + const std::string end = PrefixEnd(key); + return EtcdHelper::DeleteRange(key.data(), key.size(), end.data(), + end.size()); +} + +ErrorCode EtcdViewStore::LoadAllSegmentOwners( + const std::string& cluster_namespace, std::vector& out, + ViewVersionId& version) { + out.clear(); + const std::string prefix = SegmentViewPrefix(cluster_namespace); + const std::string end = PrefixEnd(prefix); + std::string json; + ErrorCode err = EtcdHelper::GetRangeAsJson(prefix.data(), prefix.size(), + end.data(), end.size(), + /*limit=*/0, json, version); + if (err != ErrorCode::OK) { + return err; + } + + std::vector> kvs; + err = ParseRangeJson(json, kvs); + if (err != ErrorCode::OK) { + return err; + } + + out.reserve(kvs.size()); + for (const auto& kv : kvs) { + SegmentOwner owner; + err = DeserializeSegmentOwner(kv.second, owner); + if (err != ErrorCode::OK) { + return err; + } + out.push_back(std::move(owner)); + } + return ErrorCode::OK; +} + +// ---- Master registration ---- + +ErrorCode EtcdViewStore::RegisterMaster(const std::string& cluster_namespace, + const MasterRegistration& reg, + EtcdLeaseId lease_id) { + const std::string key = + MasterRegistrationKey(cluster_namespace, reg.master_id); + std::string value; + ErrorCode err = SerializeMasterRegistration(reg, value); + if (err != ErrorCode::OK) { + return err; + } + return EtcdHelper::PutWithLease(key.data(), key.size(), value.data(), + value.size(), lease_id); +} + +ErrorCode EtcdViewStore::UpdateMasterRole(const std::string& cluster_namespace, + const std::string& master_id, + MasterRole role, + EtcdLeaseId lease_id) { + const std::string key = MasterRegistrationKey(cluster_namespace, master_id); + + MasterRegistration reg; + ViewVersionId version = 0; + std::string existing; + ErrorCode err = + EtcdHelper::Get(key.data(), key.size(), existing, version); + if (err == ErrorCode::OK) { + err = DeserializeMasterRegistration(existing, reg); + if (err != ErrorCode::OK) { + return err; + } + } + // On read failure (e.g. key missing) fall back to a minimal registration; + // the caller only flips role on a previously-registered master, so this + // path is defensive and preserves liveness via the lease. + reg.master_id = master_id; + reg.role = static_cast(role); + + std::string value; + err = SerializeMasterRegistration(reg, value); + if (err != ErrorCode::OK) { + return err; + } + return EtcdHelper::PutWithLease(key.data(), key.size(), value.data(), + value.size(), lease_id); +} + +ErrorCode EtcdViewStore::LoadAllMasters( + const std::string& cluster_namespace, std::vector& out, + ViewVersionId& version) { + out.clear(); + const std::string prefix = MasterRegistrationPrefix(cluster_namespace); + const std::string end = PrefixEnd(prefix); + std::string json; + ErrorCode err = EtcdHelper::GetRangeAsJson(prefix.data(), prefix.size(), + end.data(), end.size(), + /*limit=*/0, json, version); + if (err != ErrorCode::OK) { + return err; + } + + std::vector> kvs; + err = ParseRangeJson(json, kvs); + if (err != ErrorCode::OK) { + return err; + } + + out.reserve(kvs.size()); + for (const auto& kv : kvs) { + MasterRegistration reg; + err = DeserializeMasterRegistration(kv.second, reg); + if (err != ErrorCode::OK) { + return err; + } + out.push_back(std::move(reg)); + } + return ErrorCode::OK; +} + +// ---- Snapshots ---- + +ErrorCode EtcdViewStore::SaveKvViewSnapshot(const std::string& cluster_namespace, + const KvViewSnapshot& snapshot) { + const std::string key = KvViewSnapshotKey(cluster_namespace); + std::string value; + ErrorCode err = SerializeKvViewSnapshot(snapshot, value); + if (err != ErrorCode::OK) { + return err; + } + return EtcdHelper::Put(key.data(), key.size(), value.data(), value.size()); +} + +ErrorCode EtcdViewStore::SaveSegmentViewSnapshot( + const std::string& cluster_namespace, const SegmentViewSnapshot& snapshot) { + const std::string key = SegmentViewSnapshotKey(cluster_namespace); + std::string value; + ErrorCode err = SerializeSegmentViewSnapshot(snapshot, value); + if (err != ErrorCode::OK) { + return err; + } + return EtcdHelper::Put(key.data(), key.size(), value.data(), value.size()); +} + +namespace { +// Content-dedup cache for snapshot writes. `generated_at_ms` changes on every +// build, so a naive "write every sync cycle" put a ~MB value per cycle per +// master and exhausted the etcd backend quota with MVCC revisions. Skip the +// write when the owners content is unchanged since this process last wrote it. +// Process-local: after a content change each master writes once, which is +// acceptable (two puts per change instead of one). +std::mutex g_snapshot_dedup_mutex; +std::unordered_map g_last_snapshot_content; +} // namespace + +ErrorCode EtcdViewStore::BuildAndSaveKvViewSnapshot( + const std::string& cluster_namespace, ViewVersionId& version) { + std::vector owners; + ErrorCode err = LoadAllSlotOwners(cluster_namespace, owners, version); + if (err != ErrorCode::OK) { + return err; + } + + // Fingerprint over the owners only (exclude version/generated_at so the + // steady state produces a stable string). + std::string content; + try { + struct_json::to_json(owners, content); + } catch (const std::exception& e) { + LOG(WARNING) << "BuildAndSaveKvViewSnapshot: fingerprint failed: " + << e.what(); + content.clear(); + } + { + std::lock_guard lock(g_snapshot_dedup_mutex); + auto& last = g_last_snapshot_content["kv:" + cluster_namespace]; + if (!content.empty() && last == content) { + return ErrorCode::OK; // unchanged, skip the etcd write + } + last = content; + } + + KvViewSnapshot snapshot; + snapshot.version = version; + snapshot.generated_at_ms = + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + snapshot.slot_owners = std::move(owners); + err = SaveKvViewSnapshot(cluster_namespace, snapshot); + if (err != ErrorCode::OK) { + // 写失败时回滚缓存，下轮重试完整写入。 + std::lock_guard lock(g_snapshot_dedup_mutex); + g_last_snapshot_content.erase("kv:" + cluster_namespace); + } + return err; +} + +ErrorCode EtcdViewStore::BuildAndSaveSegmentViewSnapshot( + const std::string& cluster_namespace, ViewVersionId& version) { + std::vector owners; + ErrorCode err = LoadAllSegmentOwners(cluster_namespace, owners, version); + if (err != ErrorCode::OK) { + return err; + } + + std::string content; + try { + struct_json::to_json(owners, content); + } catch (const std::exception& e) { + LOG(WARNING) << "BuildAndSaveSegmentViewSnapshot: fingerprint failed: " + << e.what(); + content.clear(); + } + { + std::lock_guard lock(g_snapshot_dedup_mutex); + auto& last = g_last_snapshot_content["segment:" + cluster_namespace]; + if (!content.empty() && last == content) { + return ErrorCode::OK; // unchanged, skip the etcd write + } + last = content; + } + + SegmentViewSnapshot snapshot; + snapshot.version = version; + snapshot.generated_at_ms = + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + snapshot.segment_owners = std::move(owners); + err = SaveSegmentViewSnapshot(cluster_namespace, snapshot); + if (err != ErrorCode::OK) { + std::lock_guard lock(g_snapshot_dedup_mutex); + g_last_snapshot_content.erase("segment:" + cluster_namespace); + } + return err; +} + +// ---- Watch ---- + +ErrorCode EtcdViewStore::WatchKvView(const std::string& cluster_namespace, + ViewVersionId start_revision, void* ctx, + WatchCallback cb) { + const std::string prefix = KvViewPrefix(cluster_namespace); + return EtcdHelper::WatchWithPrefixFromRevision(prefix.data(), prefix.size(), + start_revision, ctx, cb); +} + +ErrorCode EtcdViewStore::CancelWatchKvView(const std::string& cluster_namespace) { + const std::string prefix = KvViewPrefix(cluster_namespace); + return EtcdHelper::CancelWatchWithPrefix(prefix.data(), prefix.size()); +} + +ErrorCode EtcdViewStore::WaitWatchKvViewStopped( + const std::string& cluster_namespace, int timeout_ms) { + const std::string prefix = KvViewPrefix(cluster_namespace); + return EtcdHelper::WaitWatchWithPrefixStopped(prefix.data(), prefix.size(), + timeout_ms); +} + +// ---- Master membership watch ---- + +ErrorCode EtcdViewStore::WatchMasters(const std::string& cluster_namespace, + ViewVersionId start_revision, void* ctx, + WatchCallback cb) { + const std::string prefix = MasterRegistrationPrefix(cluster_namespace); + return EtcdHelper::WatchWithPrefixFromRevision(prefix.data(), prefix.size(), + start_revision, ctx, cb); +} + +ErrorCode EtcdViewStore::CancelWatchMasters( + const std::string& cluster_namespace) { + const std::string prefix = MasterRegistrationPrefix(cluster_namespace); + return EtcdHelper::CancelWatchWithPrefix(prefix.data(), prefix.size()); +} + +ErrorCode EtcdViewStore::WaitWatchMastersStopped( + const std::string& cluster_namespace, int timeout_ms) { + const std::string prefix = MasterRegistrationPrefix(cluster_namespace); + return EtcdHelper::WaitWatchWithPrefixStopped(prefix.data(), prefix.size(), + timeout_ms); +} + +} // namespace cvm +} // namespace mooncake diff --git a/mooncake-store/src/cvm/inter_master_rpc.cpp b/mooncake-store/src/cvm/inter_master_rpc.cpp new file mode 100644 index 0000000000..55b9d54090 --- /dev/null +++ b/mooncake-store/src/cvm/inter_master_rpc.cpp @@ -0,0 +1,429 @@ +#include "cvm/inter_master_rpc.h" + +#include +#include + +#include +#include +#include + +#include "cvm/etcd_view_store.h" +#include "master_client.h" +#include "rpc_service.h" +#include "store_rpc_client_io_context.h" + +namespace mooncake { +namespace cvm { + +namespace { + +std::string JoinStrings(const std::vector& parts) { + std::string out; + for (const auto& p : parts) { + if (!out.empty()) { + out += ", "; + } + out += p; + } + return out; +} + +} // namespace + +InterMasterRpcClient::InterMasterRpcClient() + : pool_accessor_(GetStoreRpcClientIoContextPool(), + detail::MakeMasterRpcClientPoolConfig()) {} + +InterMasterRpcClient::~InterMasterRpcClient() { Stop(); } + +ErrorCode InterMasterRpcClient::Start(const std::string& cluster_namespace, + const std::string& self_master_id) { + if (running_.load()) { + return ErrorCode::OK; + } + if (cluster_namespace.empty()) { + LOG(WARNING) + << "InterMasterRpcClient: no cluster namespace, refresh thread " + "not started (manual member updates only)"; + return ErrorCode::INVALID_PARAMS; + } + cluster_namespace_ = cluster_namespace; + self_master_id_ = self_master_id; + running_.store(true); + refresh_thread_ = std::thread([this] { RefreshLoop(); }); + free_thread_ = std::thread([this] { FreeLoop(); }); + LOG(INFO) << "InterMasterRpcClient started: cluster_namespace=" + << cluster_namespace_ << ", self=" << self_master_id_ + << ", refresh_interval_ms=" << kRefreshIntervalMs; + return ErrorCode::OK; +} + +void InterMasterRpcClient::Stop() { + if (!running_.exchange(false)) { + return; + } + { + std::lock_guard lock(cv_mutex_); + cv_.notify_all(); + } + { + std::lock_guard lock(free_queue_mutex_); + free_cv_.notify_all(); + } + if (refresh_thread_.joinable()) { + refresh_thread_.join(); + } + if (free_thread_.joinable()) { + free_thread_.join(); + } + LOG(INFO) << "InterMasterRpcClient stopped"; +} + +void InterMasterRpcClient::UpdateMembers( + const std::vector& members) { + std::unordered_map next; + next.reserve(members.size()); + for (const auto& m : members) { + if (m.master_id.empty() || m.address.empty()) { + continue; + } + next[m.master_id] = m.address; + } + + std::lock_guard lock(members_mutex_); + if (next != members_) { + std::vector joined, left; + for (const auto& [id, addr] : next) { + if (members_.find(id) == members_.end()) { + joined.push_back(id + "@" + addr); + } + } + for (const auto& [id, addr] : members_) { + if (next.find(id) == next.end()) { + left.push_back(id + "@" + addr); + } + } + members_ = std::move(next); + LOG(INFO) << "InterMasterRpc members updated: total=" << members_.size() + << ", joined=[" << JoinStrings(joined) + << "], left=[" << JoinStrings(left) << "]"; + } +} + +std::vector InterMasterRpcClient::GetMembers() const { + std::lock_guard lock(members_mutex_); + std::vector out; + out.reserve(members_.size()); + for (const auto& [id, addr] : members_) { + MasterRegistration reg; + reg.master_id = id; + reg.address = addr; + out.push_back(std::move(reg)); + } + return out; +} + +std::optional InterMasterRpcClient::ResolveAddress( + const std::string& master_id) const { + std::lock_guard lock(members_mutex_); + auto it = members_.find(master_id); + if (it == members_.end()) { + return std::nullopt; + } + return it->second; +} + +tl::expected +InterMasterRpcClient::Handshake(const std::string& master_id) { + auto address = ResolveAddress(master_id); + if (!address) { + return tl::make_unexpected(ErrorCode::INVALID_PARAMS); + } + return invoke_rpc<&WrappedMasterService::InterMasterHandshake, + InterMasterHandshakeResponse>(*address); +} + +size_t InterMasterRpcClient::HandshakeAll(const std::string& self_master_id) { + auto members = GetMembers(); + size_t ok_count = 0; + for (const auto& m : members) { + if (m.master_id == self_master_id) { + continue; + } + auto result = Handshake(m.master_id); + if (result.has_value()) { + ++ok_count; + LOG(INFO) << "InterMasterRpc handshake ok: peer=" + << result.value().master_id + << " address=" << m.address + << " lease_id=" << result.value().lease_id + << " owned_slots=" << result.value().owned_slot_count + << " version=" << result.value().version; + } else { + LOG(WARNING) << "InterMasterRpc handshake failed: peer=" + << m.master_id << " address=" << m.address + << " error=" << toString(result.error()); + } + } + return ok_count; +} + +tl::expected, ErrorCode> +InterMasterRpcClient::AllocateReplicas( + const std::string& master_id, const std::string& tenant_id, + const std::string& key, uint64_t slice_length, uint64_t replica_num, + const std::vector& preferred_segments) { + auto address = ResolveAddress(master_id); + if (!address) { + return tl::make_unexpected(ErrorCode::INVALID_PARAMS); + } + return invoke_rpc<&WrappedMasterService::InterMasterAllocateReplicas, + std::vector>( + *address, tenant_id, key, slice_length, replica_num, + preferred_segments); +} + +tl::expected InterMasterRpcClient::FreeReplicas( + const std::string& master_id, const std::string& tenant_id, + const std::string& key) { + auto address = ResolveAddress(master_id); + if (!address) { + return tl::make_unexpected(ErrorCode::INVALID_PARAMS); + } + return invoke_rpc<&WrappedMasterService::InterMasterFreeReplicas, bool>( + *address, tenant_id, key); +} + +tl::expected +InterMasterRpcClient::GetReplicaList(const std::string& master_id, + const std::string& key, + const std::string& tenant_id) { + auto address = ResolveAddress(master_id); + if (!address) { + return tl::make_unexpected(ErrorCode::INVALID_PARAMS); + } + return invoke_rpc<&WrappedMasterService::InterMasterGetReplicaList, + GetReplicaListResponse>(*address, key, tenant_id); +} + +std::vector> +InterMasterRpcClient::BatchGetReplicaList( + const std::string& master_id, const std::vector& keys, + const std::string& tenant_id) { + auto address = ResolveAddress(master_id); + if (!address) { + std::vector> errs( + keys.size(), tl::make_unexpected(ErrorCode::INVALID_PARAMS)); + return errs; + } + // RPC 返回值本身已是逐 key 的 expected 向量；外层只关心传输层失败。 + auto result = invoke_rpc< + &WrappedMasterService::InterMasterBatchGetReplicaList, + std::vector>>( + *address, keys, tenant_id); + if (!result.has_value()) { + return std::vector>( + keys.size(), tl::make_unexpected(result.error())); + } + return std::move(result.value()); +} + +tl::expected, ErrorCode> +InterMasterRpcClient::PutStart(const std::string& master_id, + const UUID& client_id, const std::string& key, + const std::string& tenant_id, + uint64_t slice_length, + const ReplicateConfig& config) { + auto address = ResolveAddress(master_id); + if (!address) { + return tl::make_unexpected(ErrorCode::INVALID_PARAMS); + } + return invoke_rpc<&WrappedMasterService::InterMasterPutStart, + std::vector>( + *address, client_id, key, tenant_id, slice_length, config); +} + +tl::expected, ErrorCode> +InterMasterRpcClient::UpsertStart(const std::string& master_id, + const UUID& client_id, const std::string& key, + const std::string& tenant_id, + uint64_t slice_length, + const ReplicateConfig& config) { + auto address = ResolveAddress(master_id); + if (!address) { + return tl::make_unexpected(ErrorCode::INVALID_PARAMS); + } + return invoke_rpc<&WrappedMasterService::InterMasterUpsertStart, + std::vector>( + *address, client_id, key, tenant_id, slice_length, config); +} + +void InterMasterRpcClient::EnqueueBroadcastFree(const std::string& tenant_id, + const std::string& key) { + if (!running_.load()) { + VLOG(1) << "InterMasterRpc: broadcast free dropped (not running): key=" + << key; + return; + } + { + std::lock_guard lock(free_queue_mutex_); + // Bound the queue so a free storm cannot grow it unboundedly; the + // entries cover handle reclamation, so dropping only leaks until the + // segment is unmounted. + if (free_queue_.size() >= kMaxBroadcastFreeQueueSize) { + LOG(WARNING) << "InterMasterRpc: broadcast free queue full (" + << free_queue_.size() << "), dropping key=" << key; + return; + } + free_queue_.push_back(FreeTask{tenant_id, key, 0}); + } + free_cv_.notify_one(); +} + +void InterMasterRpcClient::FreeLoop() { + constexpr int kMaxFreeAttempts = 20; // ~10s when members not yet known + while (running_.load()) { + FreeTask task; + { + std::unique_lock lock(free_queue_mutex_); + free_cv_.wait(lock, [this] { + return !running_.load() || !free_queue_.empty(); + }); + if (!running_.load()) { + return; + } + task = std::move(free_queue_.front()); + free_queue_.pop_front(); + } + + auto members = GetMembers(); + if (members.empty()) { + // Member table not populated yet (startup race): retry after a + // short delay, bounded by kMaxFreeAttempts. + if (++task.attempts < kMaxFreeAttempts) { + { + std::lock_guard lock(free_queue_mutex_); + free_queue_.push_back(std::move(task)); + } + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + } else { + LOG(WARNING) + << "InterMasterRpc: broadcast free gave up (no members): " + << "key=" << task.key; + } + continue; + } + + for (const auto& member : members) { + if (member.master_id.empty() || member.master_id == self_master_id_) { + continue; + } + auto result = FreeReplicas(member.master_id, task.tenant_id, + task.key); + // Not-found (false) is the common case for non-owner peers. + if (result.has_value() && result.value()) { + LOG(INFO) << "InterMasterRpc: freed remote replicas on " + << member.master_id << " for key=" << task.key; + } else if (!result.has_value()) { + VLOG(1) << "InterMasterRpc: free failed on " + << member.master_id << " for key=" << task.key + << " error=" << toString(result.error()); + } + } + } +} + +void InterMasterRpcClient::RefreshLoop() { + while (running_.load()) { + std::vector masters; + ViewVersionId version = 0; + ErrorCode rc = EtcdViewStore::LoadAllMasters(cluster_namespace_, + masters, version); + if (rc == ErrorCode::OK) { + // Snapshot the previous table to detect newly joined peers. + std::unordered_map previous; + { + std::lock_guard lock(members_mutex_); + previous = members_; + } + UpdateMembers(masters); + + // Handshake newly joined peers: verifies the inter-master + // channel and warms up the connection pools. + std::vector joined; + for (const auto& m : masters) { + if (m.master_id.empty() || m.master_id == self_master_id_) { + continue; + } + if (previous.find(m.master_id) == previous.end()) { + joined.push_back(m.master_id); + } + } + for (const auto& peer_id : joined) { + auto result = Handshake(peer_id); + if (result.has_value()) { + LOG(INFO) << "InterMasterRpc handshake ok (new peer): peer=" + << result.value().master_id + << " lease_id=" << result.value().lease_id + << " owned_slots=" + << result.value().owned_slot_count + << " version=" << result.value().version; + } else { + LOG(WARNING) + << "InterMasterRpc handshake failed (new peer): peer=" + << peer_id << " error=" << toString(result.error()); + } + } + } else { + LOG(WARNING) << "InterMasterRpc: failed to load masters from " + "etcd: " + << toString(rc); + } + std::unique_lock lock(cv_mutex_); + cv_.wait_for(lock, std::chrono::milliseconds(kRefreshIntervalMs), + [this] { return !running_.load(); }); + } +} + +template +tl::expected InterMasterRpcClient::invoke_rpc( + const std::string& address, Args&&... args) { + auto pool = pool_accessor_.GetOrCreateClientPool(address); + auto rpc_result = async_simple::coro::syncAwait( + [&]() -> async_simple::coro::Lazy< + tl::expected> { + auto ret = co_await pool->send_request( + [&](coro_io::client_reuse_hint, + coro_rpc::coro_rpc_client& client) { + return client.send_request( + std::forward(args)...); + }); + if (!ret.has_value()) { + LOG(ERROR) << "InterMasterRpc: no available client for " + << address; + co_return tl::make_unexpected(ErrorCode::RPC_FAIL); + } + auto result = co_await std::move(ret.value()); + if (!result) { + if (result.error().code == coro_rpc::errc::timed_out) { + co_return tl::make_unexpected(ErrorCode::RPC_TIMEOUT); + } + LOG(ERROR) << "InterMasterRpc call failed on " << address + << ": " << result.error().msg; + co_return tl::make_unexpected(ErrorCode::RPC_FAIL); + } + if constexpr (std::is_void_v) { + result->result(); + co_return tl::expected{}; + } else { + co_return result->result(); + } + }()); + return rpc_result; +} + +// Explicit instantiation is not required: invoke_rpc is only used within +// this translation unit (Handshake). Future forwarding methods live here too. + +} // namespace cvm +} // namespace mooncake diff --git a/mooncake-store/src/cvm/slot_migrator.cpp b/mooncake-store/src/cvm/slot_migrator.cpp new file mode 100644 index 0000000000..38b34fd525 --- /dev/null +++ b/mooncake-store/src/cvm/slot_migrator.cpp @@ -0,0 +1,262 @@ +#include "cvm/slot_migrator.h" + +#include +#include +#include +#include + +#include + +#include "cvm/etcd_view_store.h" + +namespace mooncake { +namespace cvm { + +SlotMigrator::SlotMigrator(Config config) : config_(std::move(config)) {} + +ErrorCode SlotMigrator::PublishMigrating(uint16_t slot) { + SlotOwner owner; + owner.slot = slot; + owner.primary_master_id = config_.master_id; + owner.state = static_cast(SlotState::kMigrating); + owner.migrating_to_master_id = config_.master_id; + + if (config_.lease_id != 0) { + return EtcdViewStore::SaveSlotOwnerWithLease(config_.cluster_namespace, + owner, config_.lease_id); + } + return EtcdViewStore::SaveSlotOwner(config_.cluster_namespace, owner); +} + +ErrorCode SlotMigrator::PublishStable(uint16_t slot) { + SlotOwner owner; + owner.slot = slot; + owner.primary_master_id = config_.master_id; + owner.state = static_cast(SlotState::kStable); + // migrating_to_master_id 留空。 + + if (config_.lease_id != 0) { + return EtcdViewStore::SaveSlotOwnerWithLease(config_.cluster_namespace, + owner, config_.lease_id); + } + return EtcdViewStore::SaveSlotOwner(config_.cluster_namespace, owner); +} + +ErrorCode SlotMigrator::Reconcile(const std::vector& owned_slots) { + std::vector cur = owned_slots; + std::vector prev = last_owned_slots_; + std::sort(cur.begin(), cur.end()); + std::sort(prev.begin(), prev.end()); + + std::vector gained; + std::vector released; + std::set_difference(cur.begin(), cur.end(), prev.begin(), prev.end(), + std::back_inserter(gained)); + std::set_difference(prev.begin(), prev.end(), cur.begin(), cur.end(), + std::back_inserter(released)); + + ErrorCode last_err = ErrorCode::OK; + + // 释放：先回调清理元数据，再条件删除残留记录（确认仍指向本机才删）。 + for (uint16_t slot : released) { + if (on_release_) { + on_release_(slot); + } + ErrorCode err = EtcdViewStore::DeleteSlotOwnerIfOwnedBy( + config_.cluster_namespace, slot, config_.master_id); + if (err != ErrorCode::OK) { + LOG(WARNING) << "SlotMigrator release slot " << slot + << " failed: " << err; + last_err = err; + } + } + // 释放成功的聚合记录：仅在 owned 集合变化时打印一条，便于确认交接完成。 + if (!released.empty()) { + LOG(INFO) << "SlotMigrator released " << released.size() + << " slot(s), master_id=" << config_.master_id; + } + + // 获得：原语义为 kMigrating -> on_acquire -> kStable 两段式交接。 + // fencing：写入前检查 etcd 中现 owner。记录存在即说明原 owner 的 + // lease 仍然有效（slot key 附着在其 lease 上，lease 过期 etcd 会自动 + // 删除记录）——此时绝不抢夺，避免两个 submaster 对同一 slot 反复 + // 互相覆盖（slot 分布震荡不收敛的根因）。仅当记录已消失（原 owner + // 崩溃/lease 过期/主动释放）时才允许本机接管。 + std::vector fenced; + + // 冷启动 / 大批量认领（gained 达到门槛）：走批量路径。一次范围读完成 + // 全部 fencing（等价于逐 slot 点读，只是合并为一次往返），随后把「无 + // 前任 owner」的 slot 直接批量写 kStable（跳过 kMigrating 占位——冷启动 + // 并无交接对象，写一步足够），并把「仍被存活 peer 持有」的 slot 继续 + // fenced 观察等待。将原来 16384*(1 读 + 2 写) ≈ 4.9 万次串行往返压缩为 + // 1 次范围读 + 16384/128 ≈ 128 次批量事务写。小批量交接仍走逐 slot 路径。 + constexpr size_t kBulkClaimThreshold = 64; + auto settle_per_slot = [&](const std::vector& pending) { + for (uint16_t slot : pending) { + SlotOwner current; + ViewVersionId current_version = 0; + ErrorCode load_err = + EtcdViewStore::LoadSlotOwner(config_.cluster_namespace, slot, + current, current_version); + if (load_err == ErrorCode::OK && + !current.primary_master_id.empty() && + current.primary_master_id != config_.master_id) { + fenced.push_back(slot); + continue; + } + if (load_err != ErrorCode::OK && + load_err != ErrorCode::ETCD_KEY_NOT_EXIST) { + // etcd 读取异常：保守跳过，待下轮重试，不做盲目覆盖。 + fenced.push_back(slot); + last_err = load_err; + LOG(WARNING) << "SlotMigrator fence check slot " << slot + << " failed: " << load_err + << ", master_id=" << config_.master_id; + continue; + } + + ErrorCode err = PublishMigrating(slot); + if (err != ErrorCode::OK) { + LOG(WARNING) << "SlotMigrator publish migrating slot " << slot + << " failed: " << err; + last_err = err; + fenced.push_back(slot); // 发布失败同样下轮重试 + continue; + } + if (on_acquire_) { + on_acquire_(slot); + } + err = PublishStable(slot); + if (err != ErrorCode::OK) { + LOG(WARNING) << "SlotMigrator publish stable slot " << slot + << " failed: " << err; + last_err = err; + } + } + }; + + if (gained.size() >= kBulkClaimThreshold) { + // ---- 批量路径：1 次范围读做 fence ---- + std::vector all; + ViewVersionId range_version = 0; + ErrorCode range_err = EtcdViewStore::LoadAllSlotOwners( + config_.cluster_namespace, all, range_version); + if (range_err == ErrorCode::OK) { + std::unordered_map owner_map; + owner_map.reserve(all.size()); + for (const auto& o : all) { + owner_map.emplace(o.slot, o.primary_master_id); + } + + std::vector claim; + claim.reserve(gained.size()); + for (uint16_t slot : gained) { + auto it = owner_map.find(slot); + const bool live_other = + (it != owner_map.end() && !it->second.empty() && + it->second != config_.master_id); + if (live_other) { + fenced.push_back(slot); + continue; + } + SlotOwner owner; + owner.slot = slot; + owner.primary_master_id = config_.master_id; + owner.state = static_cast(SlotState::kStable); + claim.push_back(owner); + } + + for (const auto& o : claim) { + if (on_acquire_) { + on_acquire_(o.slot); + } + } + if (!claim.empty()) { + ErrorCode err = EtcdViewStore::SaveSlotOwnersWithLease( + config_.cluster_namespace, claim, config_.lease_id); + if (err != ErrorCode::OK) { + LOG(WARNING) << "SlotMigrator bulk claim " + << claim.size() << " slot(s) failed: " << err + << ", master_id=" << config_.master_id; + last_err = err; + // 批量失败：整批视为 fenced，下轮重试，避免误认为已落 + // 盘而跳过 Reconcile。 + for (const auto& o : claim) { + fenced.push_back(o.slot); + } + } + } + } else { + // 范围读失败（etcd 异常）：保守退回逐 slot 路径，不盲目批量覆盖。 + LOG(WARNING) << "SlotMigrator bulk fence read failed: " << range_err + << ", falling back to per-slot, master_id=" + << config_.master_id; + last_err = range_err; + settle_per_slot(gained); + } + } else { + settle_per_slot(gained); + } + if (!gained.empty()) { + size_t acquired = gained.size(); + if (acquired >= fenced.size()) { + acquired -= fenced.size(); // 仅统计真正落盘的 slot + } else { + acquired = 0; + } + if (acquired > 0) { + // 获得成功的聚合记录：确认所有权已全部落盘。 + LOG(INFO) << "SlotMigrator acquired " << acquired + << " slot(s), master_id=" << config_.master_id; + } + if (!fenced.empty()) { + // fencing / 写失败汇总：这些 slot 仍由其他存活 master 持有，或 + // 本机写入失败，观察等待下轮重试后再接管。 + LOG(INFO) << "SlotMigrator fenced/retry " << fenced.size() + << " slot(s), master_id=" << config_.master_id; + } + } + + // 不变 slot：幂等 reaffirm kStable，但仅作为低频安全网（每 + // kReaffirmIntervalCycles 个周期一次）。slot key 附着在 lease 上， + // keepalive 持续保活即不过期；逐周期重写等值 value 只会线性堆积 etcd + // MVCC revision（曾以 ~3k put/s 的速率写满 backend 配额）。正常稳态 + // 下 lease 存活即代表所有权持续有效，无需重写。 + constexpr uint64_t kReaffirmIntervalCycles = 12; // ~60s @ 5s heartbeat + const bool do_reaffirm = (++reconcile_cycles_ % kReaffirmIntervalCycles) == 0; + if (do_reaffirm) { + for (uint16_t slot : cur) { + // fenced slot（本周期被拦截/发布失败）不能 reaffirm：它仍归 + // 其他存活 master 所有，重写会重新引发覆盖。 + if (std::binary_search(fenced.begin(), fenced.end(), slot)) { + continue; + } + if (std::binary_search(gained.begin(), gained.end(), slot)) { + continue; + } + ErrorCode err = PublishStable(slot); + if (err != ErrorCode::OK) { + LOG(WARNING) << "SlotMigrator reaffirm slot " << slot + << " failed: " << err; + last_err = err; + } + } + } + + // last_owned_slots_ 只记录本周期真正发布成功的 slot；fenced slot 不计入， + // 下一周期它们重新进入 gained 集合，重新走 fence 检查 → 观察等待。 + if (!fenced.empty()) { + std::sort(fenced.begin(), fenced.end()); + std::vector committed; + committed.reserve(cur.size() - fenced.size()); + std::set_difference(cur.begin(), cur.end(), fenced.begin(), + fenced.end(), std::back_inserter(committed)); + last_owned_slots_ = std::move(committed); + } else { + last_owned_slots_ = std::move(cur); + } + return last_err; +} + +} // namespace cvm +} // namespace mooncake diff --git a/mooncake-store/src/cvm/slot_owner_heartbeat.cpp b/mooncake-store/src/cvm/slot_owner_heartbeat.cpp new file mode 100644 index 0000000000..d8d83eac65 --- /dev/null +++ b/mooncake-store/src/cvm/slot_owner_heartbeat.cpp @@ -0,0 +1,96 @@ +#include "cvm/slot_owner_heartbeat.h" + +#include +#include +#include + +#include + +#include "cvm/etcd_view_store.h" +#include "cvm/slot_hash.h" + +namespace mooncake { +namespace cvm { + +namespace { + +// Resolves the owned slot list from config. An empty `owned_slots` means the +// submaster owns every logical slot (single-master mode). +std::vector ResolveOwnedSlots(const SlotOwnerHeartbeat::Config& cfg) { + if (!cfg.owned_slots.empty()) { + return cfg.owned_slots; + } + std::vector slots; + slots.reserve(kSlotCount); + for (uint16_t slot = 0; slot < kSlotCount; ++slot) { + slots.push_back(slot); + } + return slots; +} + +} // namespace + +SlotOwnerHeartbeat::SlotOwnerHeartbeat(Config config) + : config_(std::move(config)), + owned_slots_(ResolveOwnedSlots(config_)), + migrator_(SlotMigrator::Config{config_.cluster_namespace, + config_.master_id, + config_.lease_id}) { + migrator_.SetOnAcquire(config_.on_slot_acquired); + migrator_.SetOnRelease(config_.on_slot_released); +} + +SlotOwnerHeartbeat::~SlotOwnerHeartbeat() { Stop(); } + +ErrorCode SlotOwnerHeartbeat::Start() { + if (running_.load()) { + return ErrorCode::OK; + } + running_.store(true); + thread_ = std::thread([this]() { RunLoop(); }); + return ErrorCode::OK; +} + +void SlotOwnerHeartbeat::Stop() { + if (!running_.exchange(false)) { + return; + } + stop_cv_.notify_all(); + if (thread_.joinable()) { + thread_.join(); + } +} + +ErrorCode SlotOwnerHeartbeat::PublishOnce() { + // Under dynamic partition the owned slot set is recomputed every publish so + // it follows cluster membership changes; otherwise use the static set. + std::vector slots = owned_slots_; + if (config_.dynamic_slot_resolver) { + slots = config_.dynamic_slot_resolver(); + } + + // Delegate the per-slot writes to the SlotMigrator state machine: it + // publishes kMigrating -> kStable for newly-acquired slots, deletes + // released slots, and re-affirms unchanged slots. + ErrorCode err = migrator_.Reconcile(slots); + if (err != ErrorCode::OK) { + // Reconcile 内部已有逐 slot 失败 WARNING；这里补一条周期级汇总， + // 便于从心跳线程视角确认本轮发布未完全成功。 + LOG(WARNING) << "SlotOwnerHeartbeat publish incomplete: master_id=" + << config_.master_id << " owned_slots=" << slots.size() + << " err=" << err; + } + return err; +} + +void SlotOwnerHeartbeat::RunLoop() { + while (running_.load()) { + (void)PublishOnce(); + std::unique_lock lock(stop_mutex_); + stop_cv_.wait_for(lock, config_.heartbeat_interval, + [this] { return !running_.load(); }); + } +} + +} // namespace cvm +} // namespace mooncake diff --git a/mooncake-store/src/etcd_helper.cpp b/mooncake-store/src/etcd_helper.cpp index 29cd832a08..4888d45ffe 100644 --- a/mooncake-store/src/etcd_helper.cpp +++ b/mooncake-store/src/etcd_helper.cpp @@ -71,8 +71,11 @@ ErrorCode EtcdHelper::Get(const char* key, const size_t key_size, EtcdStoreGetWrapper(const_cast(key), (int)key_size, &value_ptr, &value_size, &revision_id, &err_msg); if (ret == -2) { - LOG(ERROR) << "key=" << std::string(key, key_size) - << ", error=" << err_msg; + // Key-not-found is an expected outcome for existence probes (e.g. + // ImportSlotMetadata checking for a graceful export). Downgrade to + // VLOG to avoid ERROR-level spam for benign misses. + VLOG(1) << "key=" << std::string(key, key_size) + << ", error=" << err_msg; free(err_msg); return ErrorCode::ETCD_KEY_NOT_EXIST; } @@ -151,8 +154,54 @@ ErrorCode EtcdHelper::BatchCreate(const std::vector& keys, return ErrorCode::OK; } +ErrorCode EtcdHelper::BatchPutWithLease(const std::vector& keys, + const std::vector& values, + EtcdLeaseId lease_id) { + if (keys.size() != values.size()) { + return ErrorCode::INVALID_PARAMS; + } + if (keys.empty()) { + return ErrorCode::OK; + } + + std::vector c_keys; + std::vector c_key_sizes; + std::vector c_values; + std::vector c_value_sizes; + c_keys.reserve(keys.size()); + c_key_sizes.reserve(keys.size()); + c_values.reserve(values.size()); + c_value_sizes.reserve(values.size()); + + for (const auto& key : keys) { + c_keys.push_back(const_cast(key.data())); + c_key_sizes.push_back(static_cast(key.size())); + } + for (const auto& val : values) { + c_values.push_back(const_cast(val.data())); + c_value_sizes.push_back(static_cast(val.size())); + } + + char* err_msg = nullptr; + int ret = EtcdStoreBatchPutWithLeaseWrapper( + c_keys.data(), c_key_sizes.data(), c_values.data(), c_value_sizes.data(), + static_cast(keys.size()), lease_id, &err_msg); + if (ret != 0) { + LOG(ERROR) << "BatchPutWithLease failed (count=" << keys.size() + << ", lease=" << lease_id + << "): " << (err_msg == nullptr ? "" : err_msg); + if (err_msg != nullptr) { + free(err_msg); + } + return ErrorCode::ETCD_OPERATION_ERROR; + } + return ErrorCode::OK; +} + ErrorCode EtcdHelper::TxnCompareAndPut(const std::vector& compares, - const std::vector& puts) { + const std::vector& puts, + const std::vector& + delete_keys) { std::vector compare_keys; std::vector compare_key_sizes; std::vector compare_kinds; @@ -187,6 +236,14 @@ ErrorCode EtcdHelper::TxnCompareAndPut(const std::vector& compares, put_values.push_back(const_cast(put.value.data())); put_value_sizes.push_back(static_cast(put.value.size())); } + std::vector deletes; + std::vector delete_sizes; + deletes.reserve(delete_keys.size()); + delete_sizes.reserve(delete_keys.size()); + for (const auto& key : delete_keys) { + deletes.push_back(const_cast(key.data())); + delete_sizes.push_back(static_cast(key.size())); + } char* err_msg = nullptr; int ret = EtcdStoreTxnCompareAndPutWrapper( @@ -194,7 +251,8 @@ ErrorCode EtcdHelper::TxnCompareAndPut(const std::vector& compares, compare_values.data(), compare_value_sizes.data(), static_cast(compares.size()), put_keys.data(), put_key_sizes.data(), put_values.data(), put_value_sizes.data(), - static_cast(puts.size()), &err_msg); + static_cast(puts.size()), deletes.data(), delete_sizes.data(), + static_cast(delete_keys.size()), &err_msg); if (ret == -2) { if (err_msg != nullptr) { free(err_msg); @@ -537,6 +595,16 @@ ErrorCode EtcdHelper::BatchCreate(const std::vector& keys, return ErrorCode::ETCD_OPERATION_ERROR; } +ErrorCode EtcdHelper::BatchPutWithLease(const std::vector& keys, + const std::vector& values, + EtcdLeaseId lease_id) { + (void)keys; + (void)values; + (void)lease_id; + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + ErrorCode EtcdHelper::GrantLease(int64_t lease_ttl, EtcdLeaseId& lease_id) { (void)lease_ttl; (void)lease_id; @@ -617,9 +685,12 @@ ErrorCode EtcdHelper::Create(const char* key, const size_t key_size, } ErrorCode EtcdHelper::TxnCompareAndPut(const std::vector& compares, - const std::vector& puts) { + const std::vector& puts, + const std::vector& + delete_keys) { (void)compares; (void)puts; + (void)delete_keys; LOG(FATAL) << "Etcd is not enabled in compilation"; return ErrorCode::ETCD_OPERATION_ERROR; } diff --git a/mooncake-store/src/file_storage.cpp b/mooncake-store/src/file_storage.cpp index 02f22968c4..c637b096c6 100644 --- a/mooncake-store/src/file_storage.cpp +++ b/mooncake-store/src/file_storage.cpp @@ -543,7 +543,6 @@ tl::expected FileStorage::OffloadObjects( } buckets_keys.emplace_back(std::move(keys)); } - auto complete_handler = [this, &task_by_storage_key]( const std::vector& keys, @@ -816,6 +815,16 @@ tl::expected FileStorage::IsEnableOffloading() { return enable_offloading; } +tl::expected FileStorage::MarkRemoved( + const std::string& key) { + return storage_backend_->MarkRemoved(key); +} + +tl::expected FileStorage::BatchMarkRemoved( + const std::vector& keys) { + return storage_backend_->BatchMarkRemoved(keys); +} + tl::expected FileStorage::Heartbeat() { if (client_ == nullptr) { LOG(ERROR) << "client is nullptr"; @@ -842,6 +851,41 @@ tl::expected FileStorage::Heartbeat() { }); } + // === STEP 0: Drain removed keys from master === + // Master pushes {tenant_id, key} pairs to this client's removed_keys + // queue when a Remove/BatchRemove deletes a key that had a LOCAL_DISK + // replica here. We mark each as a tombstone so GC can reclaim SSD space. + { + auto remove_result = + client_->RemoveObjectHeartbeat(client_->getClientId()); + if (remove_result) { + bool all_marked = true; + for (const auto& item : remove_result.value()) { + auto storage_key = + TenantId(item.tenant_id).MakeScopedKey(item.key); + auto mark_result = storage_backend_->MarkRemoved(storage_key); + if (!mark_result) { + all_marked = false; + LOG(ERROR) << "Failed to persist remove tombstone: " + << mark_result.error(); + break; + } + } + if (all_marked && !remove_result.value().empty()) { + auto ack_result = client_->AckRemoveObjectHeartbeat( + client_->getClientId(), remove_result.value()); + if (!ack_result) { + LOG(ERROR) << "Failed to ACK remove tasks: " + << ack_result.error(); + } + VLOG(1) << "RemoveObjectHeartbeat processed " + << remove_result.value().size() + << " removed key(s) from master"; + } + } + // Errors are non-fatal: removed keys will be retried next heartbeat. + } + std::vector offloading_objects; // Objects selected for offloading diff --git a/mooncake-store/src/ha/leadership/master_service_supervisor.cpp b/mooncake-store/src/ha/leadership/master_service_supervisor.cpp index 4b94942e89..9a01f09266 100644 --- a/mooncake-store/src/ha/leadership/master_service_supervisor.cpp +++ b/mooncake-store/src/ha/leadership/master_service_supervisor.cpp @@ -2,17 +2,20 @@ #include #include +#include #include #include +#include #include +#include #include +#include #include #include #include #include -#include "ha/leadership/leader_coordinator_factory.h" #include "ha/leadership/leader_label_reconciler.h" #include "ha/master_metrics_reporter.h" #include "ha/standby_controller.h" @@ -21,15 +24,18 @@ #include "rpc_service.h" #include "types.h" +#include "cvm/cvm_controller.h" +#include "cvm/cvm_service_delegate.h" +#include "cvm/etcd_view_store.h" +#include "etcd_helper.h" + namespace mooncake { namespace ha { namespace { -constexpr auto kAcquireRetryInterval = std::chrono::seconds(1); -constexpr auto kRenewCheckInterval = std::chrono::seconds(1); -constexpr auto kSupervisorRetryInterval = std::chrono::seconds(1); constexpr auto kLabelReconcileRetryInterval = std::chrono::seconds(1); +constexpr auto kSupervisorRetryInterval = std::chrono::seconds(1); constexpr char kLeaderLabelKey[] = "mooncake.io/store-role"; constexpr char kLeaderLabelValue[] = "leader"; @@ -88,14 +94,6 @@ bool IsFatalHABackendError(ErrorCode err) { err == ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; } -void LogLeadershipReleaseWarning(std::string_view context, ErrorCode err) { - if (err == ErrorCode::OK) { - return; - } - LOG(WARNING) << "Failed to release leadership after " << context << ": " - << toString(err); -} - bool HandleSupervisorError(std::string_view action, ErrorCode err, HABackendType backend_type) { if (IsFatalHABackendError(err)) { @@ -111,46 +109,6 @@ bool HandleSupervisorError(std::string_view action, ErrorCode err, return false; } -bool HandleLeadershipPhaseError(std::string_view release_context, - std::string_view action, - LeaderCoordinator& coordinator, - const LeadershipSession& session, ErrorCode err, - HABackendType backend_type) { - LogLeadershipReleaseWarning(release_context, - coordinator.ReleaseLeadership(session)); - return HandleSupervisorError(action, err, backend_type); -} - -tl::expected WarmupLeadership( - LeaderCoordinator& coordinator, const LeadershipSession& session) { - const auto deadline = std::chrono::steady_clock::now() + session.lease_ttl; - const auto max_sleep_interval = - std::chrono::duration_cast( - kRenewCheckInterval); - - while (true) { - auto renewed = coordinator.RenewLeadership(session); - if (!renewed) { - return tl::make_unexpected(renewed.error()); - } - if (!renewed.value()) { - return false; - } - - const auto now = std::chrono::steady_clock::now(); - if (now >= deadline) { - return true; - } - - const auto remaining = - std::chrono::duration_cast(deadline - - now); - const auto sleep_interval = - remaining < max_sleep_interval ? remaining : max_sleep_interval; - std::this_thread::sleep_for(sleep_interval); - } -} - void SetRuntimeState(MasterAdminServer& admin_server, MasterRuntimeState state) { admin_server.SetRuntimeState(state); @@ -174,40 +132,31 @@ void DeactivateServingState(MasterAdminServer& admin_server, label_reconciler.SetLeader(false); } -void StopLeadershipMonitor(std::unique_ptr& monitor) { - if (!monitor) { - return; - } - monitor->Stop(); - monitor.reset(); -} - void UpdateObservedLeader(MasterAdminServer& admin_server, StandbyController& standby_controller, - const std::optional& leader_view) { + const std::optional& leader_view, + const MasterSources& sources) { admin_server.SetObservedLeader(leader_view); - standby_controller.UpdateObservedLeader(leader_view); -} - -void ApplyCurrentView(MasterAdminServer& admin_server, - StandbyController& standby_controller, - const ViewChangeResult& wait_result) { - if (!wait_result.current_view.has_value() && - (wait_result.changed || wait_result.timed_out)) { - return; - } - UpdateObservedLeader(admin_server, standby_controller, - wait_result.current_view); + standby_controller.UpdateObservedLeader(sources); } -void EnterStandbyMode(MasterAdminServer& admin_server, - StandbyController& standby_controller, - std::atomic& accept_runtime_updates, - const std::optional& leader_view) { +// Forward declarations: definitions live below the CVM membership bridge. +std::optional BuildCvmObservedLeader( + const std::unique_ptr& cvm_controller); +MasterSources BuildCvmObservedSources( + const std::unique_ptr& cvm_controller); + +void EnterStandbyMode( + MasterAdminServer& admin_server, + StandbyController& standby_controller, + std::atomic& accept_runtime_updates, + const std::unique_ptr& cvm_controller) { accept_runtime_updates.store(true, std::memory_order_release); - UpdateObservedLeader(admin_server, standby_controller, leader_view); + const MasterSources sources = BuildCvmObservedSources(cvm_controller); + UpdateObservedLeader(admin_server, standby_controller, + BuildCvmObservedLeader(cvm_controller), sources); - auto err = standby_controller.StartStandby(leader_view); + auto err = standby_controller.StartStandby(sources); if (err != ErrorCode::OK) { LOG(WARNING) << "Failed to start standby replication: " << toString(err); @@ -218,6 +167,135 @@ void EnterStandbyMode(MasterAdminServer& admin_server, SetRuntimeState(admin_server, standby_controller.GetStandbyRuntimeState()); } +// Bridges CvmController's membership role decisions to the supervisor's +// serving/standby state machine. CvmController::MembershipLoop runs on its own +// thread and calls OnRoleChanged; this class persists the role back to etcd +// (so slot partitioning immediately excludes demoted nodes) and publishes the +// change to the supervisor main loop. It never mutates the supervisor state +// machine directly: on demotion it only invokes a lightweight stop-server +// signal installed by the serve phase, and the main loop performs the full +// downgrade sequence serially after the server unblocks. +class CvmMembershipCoordinator : public cvm::CvmServiceDelegate { + public: + CvmMembershipCoordinator(std::string cluster_namespace, + std::string master_id) + : cluster_namespace_(std::move(cluster_namespace)), + master_id_(std::move(master_id)) {} + + void SetLeaseId(EtcdLeaseId lease_id) { + lease_id_.store(lease_id, std::memory_order_relaxed); + } + + // Installed by the serve phase; cleared (empty) when not serving. The + // signal only stops the coro_rpc server so the serve phase unblocks; the + // main loop then reads CvmController::GetCurrentRole() and performs the + // full downgrade sequence. + void SetServeStopSignal(std::function signal) { + std::lock_guard lock(mutex_); + stop_serve_signal_ = std::move(signal); + } + + // Blocks until either a role change or a kv-view change is pending, then + // clears both so each change is consumed exactly once. The caller re-reads + // CvmController::GetCurrentRole() as the source of truth after waking; a + // standby uses the wake-up to re-bind its replay sources. + void WaitForRoleOrViewChange() { + std::unique_lock lock(mutex_); + cv_.wait(lock, [this] { + return has_pending_role_ || has_pending_view_; + }); + has_pending_role_ = false; + has_pending_view_ = false; + } + + void OnSlotAcquired(uint16_t /*slot*/) override {} + void OnSlotReleased(uint16_t /*slot*/) override {} + + void OnKvViewChanged() override { + { + std::lock_guard lock(mutex_); + has_pending_view_ = true; + } + cv_.notify_all(); + } + + void OnRoleChanged(cvm::MasterRole new_role) override { + const EtcdLeaseId lease_id = + lease_id_.load(std::memory_order_relaxed); + ErrorCode err = cvm::EtcdViewStore::UpdateMasterRole( + cluster_namespace_, master_id_, new_role, lease_id); + if (err != ErrorCode::OK) { + LOG(WARNING) << "CvmMembershipCoordinator: failed to persist role=" + << static_cast(new_role) + << " for master_id=" << master_id_ << ": " << err; + } + LOG(INFO) << "CvmMembershipCoordinator::OnRoleChanged: master_id=" + << master_id_ << ", role=" << static_cast(new_role) + << ", lease_id=" << lease_id; + + { + std::lock_guard lock(mutex_); + has_pending_role_ = true; + } + cv_.notify_all(); + + if (new_role == cvm::MasterRole::kStandby) { + std::function signal; + { + std::lock_guard lock(mutex_); + signal = stop_serve_signal_; + } + if (signal) { + signal(); + } + } + } + + private: + std::string cluster_namespace_; + std::string master_id_; + std::atomic lease_id_{0}; + std::mutex mutex_; + std::condition_variable cv_; + bool has_pending_role_{false}; + bool has_pending_view_{false}; + std::function stop_serve_signal_; +}; + +// Builds the representative single-leader view for the admin surface. In the +// multi-submaster model there is no single "leader"; this reports the earliest +// primary for display only. Returns nullopt when no primary is available yet +// (or this node is the first primary itself, so there is no upstream). +std::optional BuildCvmObservedLeader( + const std::unique_ptr& cvm_controller) { + if (!cvm_controller) { + return std::nullopt; + } + const std::string primary = cvm_controller->GetPrimaryAddress(); + if (primary.empty()) { + return std::nullopt; + } + MasterView view; + view.leader_address = primary; + view.view_version = cvm_controller->GetKvViewVersion(); + return view; +} + +// Builds the standby replay source set from the CVM membership ranking. Under +// dynamic binding (2c) a standby only follows the primary(s) that own the slot +// range it is responsible for, instead of following every primary (2b). +MasterSources BuildCvmObservedSources( + const std::unique_ptr& cvm_controller) { + MasterSources sources; + if (!cvm_controller) { + return sources; + } + for (const auto& member : cvm_controller->GetBindingSources()) { + sources.push_back(MasterSource{member.master_id, member.address}); + } + return sources; +} + int RunSupervisorLoop(const HABackendSpec& spec, const MasterServiceSupervisorConfig& config, MasterAdminServer& admin_server) { @@ -247,272 +325,209 @@ int RunSupervisorLoop(const HABackendSpec& spec, SetRuntimeState(admin_server, state); }); - EnterStandbyMode(admin_server, *standby_controller, - accept_standby_runtime_updates, std::nullopt); - - while (true) { - auto coordinator = CreateLeaderCoordinator(spec); - if (!coordinator) { - if (HandleSupervisorError("create leader coordinator", - coordinator.error(), spec.type)) { - return -1; + // ── CVM membership (quota coordination) ── + // The CvmController owns the etcd lease, master registration, view + // snapshot aggregation and the membership loop that decides this node's + // role (primary vs standby) via first-come-first-served ranking. It lives + // here (supervisor layer) so membership keeps running whether or not this + // node is currently serving; MasterService only publishes slot ownership. + // NOTE: the coordinator is declared before the controller so it is + // destroyed after it — the controller's Stop() joins the membership thread, + // guaranteeing no OnRoleChanged callback is in flight when the coordinator + // is torn down. + CvmMembershipCoordinator cvm_membership_coordinator(config.cluster_id, + config.local_hostname); + std::unique_ptr cvm_controller; + if (spec.type == HABackendType::ETCD && !config.local_hostname.empty() && + !config.cluster_id.empty()) { + ErrorCode connect_err = EtcdHelper::ConnectToEtcdStoreClient( + ResolveHABackendConnstring(config)); + if (connect_err != ErrorCode::OK) { + LOG(WARNING) << "CVM membership disabled: failed to connect etcd: " + << connect_err; + } else { + cvm::CvmController::Config cc_config; + cc_config.cluster_namespace = config.cluster_id; + cc_config.master_id = config.local_hostname; + cc_config.address = config.local_hostname; + // Start as standby; the membership loop promotes to primary when + // this node ranks within the submaster quota. + cc_config.role = cvm::MasterRole::kStandby; + cc_config.http_port = config.cvm_http_port; + cc_config.http_host = config.cvm_http_host; + cc_config.submaster_count = config.submaster_count; + cvm_controller = + std::make_unique(std::move(cc_config)); + cvm_controller->SetDelegate(&cvm_membership_coordinator); + ErrorCode cc_err = cvm_controller->Start(); + if (cc_err != ErrorCode::OK) { + LOG(WARNING) << "Failed to start CvmController: " << cc_err; + cvm_controller.reset(); + } else { + cvm_membership_coordinator.SetLeaseId( + cvm_controller->GetLeaseId()); + LOG(INFO) << "Started supervisor-owned CvmController: master_id=" + << config.local_hostname + << ", cluster_namespace=" << config.cluster_id + << ", lease_id=" << cvm_controller->GetLeaseId() + << ", submaster_count=" << config.submaster_count; } - continue; } + } - auto& leader_coordinator = *coordinator.value(); - std::optional leadership_session; - - while (!leadership_session.has_value()) { - SetRuntimeState(admin_server, MasterRuntimeState::kCandidate); - - auto current_view = leader_coordinator.ReadCurrentView(); - if (!current_view) { - EnterStandbyMode(admin_server, *standby_controller, - accept_standby_runtime_updates, std::nullopt); - if (HandleSupervisorError("read current leader view", - current_view.error(), spec.type)) { - return -1; - } - break; - } + EnterStandbyMode(admin_server, *standby_controller, + accept_standby_runtime_updates, + cvm_controller); - UpdateObservedLeader(admin_server, *standby_controller, - current_view.value()); - if (!current_view.value().has_value()) { - auto acquire = leader_coordinator.TryAcquireLeadership( - config.local_hostname); - if (!acquire) { + while (true) { + // CVM membership decides this node's role via first-come-first-served + // ranking against the submaster quota. Without a CvmController + // (non-etcd backend or missing identity) there is no quota + // coordination, so fall back to unconditional primary serving. + const cvm::MasterRole target_role = + cvm_controller ? cvm_controller->GetCurrentRole() + : cvm::MasterRole::kPrimary; + + if (target_role == cvm::MasterRole::kPrimary) { + // ── Upgrade sequence (kStandby → kPrimary) ── + // first primary（无上游）无需 final catch-up；有上游 primary 时才 + // 从 standby 导出回放数据。PromoteStandbyAndExport 内部完成 final + // catch-up + 导出 PromotionContext，新 primary 从它恢复。 + accept_standby_runtime_updates.store(false, + std::memory_order_release); + const bool has_upstream = + cvm_controller && !cvm_controller->GetPrimaryAddress().empty(); + PromotionContext promotion_ctx{}; + if (has_upstream) { + auto ctx = standby_controller->PromoteStandbyAndExport(); + if (!ctx) { EnterStandbyMode(admin_server, *standby_controller, accept_standby_runtime_updates, - std::nullopt); - if (HandleSupervisorError("acquire leadership", - acquire.error(), spec.type)) { - return -1; - } - break; - } - - if (acquire->observed_view.has_value()) { - UpdateObservedLeader(admin_server, *standby_controller, - acquire->observed_view); - } - - if (acquire->status == AcquireLeadershipStatus::ACQUIRED && - acquire->session.has_value()) { - leadership_session = *acquire->session; - admin_server.SetObservedLeader(leadership_session->view); - break; - } - - EnterStandbyMode(admin_server, *standby_controller, - accept_standby_runtime_updates, - acquire->observed_view); - std::optional observed_version = std::nullopt; - if (acquire->observed_view.has_value()) { - observed_version = acquire->observed_view->view_version; - } - auto wait = leader_coordinator.WaitForViewChange( - observed_version, kAcquireRetryInterval); - if (!wait) { - if (HandleSupervisorError("wait for leader view change", - wait.error(), spec.type)) { + cvm_controller); + if (HandleSupervisorError("promote standby for serve", + ctx.error(), spec.type)) { return -1; } - break; + continue; } - ApplyCurrentView(admin_server, *standby_controller, *wait); - continue; + promotion_ctx = std::move(*ctx); } - EnterStandbyMode(admin_server, *standby_controller, - accept_standby_runtime_updates, - current_view.value()); - const auto& known_view = current_view.value().value(); - auto wait = leader_coordinator.WaitForViewChange( - known_view.view_version, kAcquireRetryInterval); - if (!wait) { - if (HandleSupervisorError("wait for leader view change", - wait.error(), spec.type)) { - return -1; - } - break; + LOG(INFO) << "Starting serve phase (CVM role=kPrimary)..."; + coro_rpc::coro_rpc_server server( + config.rpc_thread_num, config.rpc_port, config.rpc_address, + config.rpc_conn_timeout, config.rpc_enable_tcp_no_delay); + const char* protocol = std::getenv("MC_RPC_PROTOCOL"); + if (protocol && std::string_view(protocol) == "rdma") { + server.init_ibv(); } - ApplyCurrentView(admin_server, *standby_controller, *wait); - } - - if (!leadership_session.has_value()) { - continue; - } - accept_standby_runtime_updates.store(false, std::memory_order_release); - auto promotion_ctx = standby_controller->PromoteStandbyAndExport(); - if (!promotion_ctx) { - EnterStandbyMode(admin_server, *standby_controller, - accept_standby_runtime_updates, - leadership_session->view); - if (HandleSupervisorError("promote standby for serve", - promotion_ctx.error(), spec.type)) { - return -1; + const ViewVersionId view_version = + cvm_controller ? cvm_controller->GetKvViewVersion() : 0; + mooncake::WrappedMasterServiceConfig wrapped_config(config, + view_version); + // In HA serving-primary mode, snapshot bootstrap belongs to + // standby. The new primary must restore from PromotionContext only. + wrapped_config.enable_snapshot_restore = false; + auto wrapped_master_service = + std::make_shared( + wrapped_config, config.http_metadata_server, + config.http_metadata_remote_url); + + // Inject the supervisor-owned CvmController lease so the + // slot/segment ownership records share this master's registration + // lifecycle. + if (cvm_controller) { + wrapped_master_service->SetCvmLeaseId( + cvm_controller->GetLeaseId()); } - continue; - } - LOG(INFO) << "Entering warmup phase..."; - SetRuntimeState(admin_server, MasterRuntimeState::kLeaderWarmup); - auto warmup_result = - WarmupLeadership(leader_coordinator, *leadership_session); - if (!warmup_result) { - EnterStandbyMode(admin_server, *standby_controller, - accept_standby_runtime_updates, - leadership_session->view); - if (HandleLeadershipPhaseError( - "renewal startup failure", "start leadership renewal", - leader_coordinator, *leadership_session, - warmup_result.error(), spec.type)) { - return -1; + // Restore from standby if we have context. + if (promotion_ctx.applied_seq_id > 0 || + !promotion_ctx.objects.empty() || + !promotion_ctx.segments.empty()) { + wrapped_master_service->RestoreFromStandby( + promotion_ctx.objects, promotion_ctx.applied_seq_id, + promotion_ctx.segments); } - continue; - } - if (!warmup_result.value()) { - EnterStandbyMode(admin_server, *standby_controller, - accept_standby_runtime_updates, std::nullopt); - LogLeadershipReleaseWarning( - "warmup expiration", - leader_coordinator.ReleaseLeadership(*leadership_session)); - LOG(WARNING) << "Leadership expired during warmup phase"; - admin_server.SetObservedLeader(std::nullopt); - continue; - } - - LOG(INFO) << "Starting serve phase..."; - coro_rpc::coro_rpc_server server( - config.rpc_thread_num, config.rpc_port, config.rpc_address, - config.rpc_conn_timeout, config.rpc_enable_tcp_no_delay); - const char* protocol = std::getenv("MC_RPC_PROTOCOL"); - if (protocol && std::string_view(protocol) == "rdma") { - server.init_ibv(); - } - - mooncake::WrappedMasterServiceConfig wrapped_config( - config, leadership_session->view.view_version); - // In HA serving-primary mode, snapshot bootstrap belongs to standby. - // The new primary must restore from PromotionContext only. - wrapped_config.enable_snapshot_restore = false; - // The serving primary handles heartbeats/unmounts, so forward the - // metadata cleanup config here like the non-HA path does. - auto wrapped_master_service = std::make_shared( - wrapped_config, config.http_metadata_server, - config.http_metadata_remote_url); - - // Restore from standby if we have context - if (promotion_ctx->applied_seq_id > 0 || - !promotion_ctx->objects.empty() || - !promotion_ctx->segments.empty()) { - wrapped_master_service->RestoreFromStandby( - promotion_ctx->objects, promotion_ctx->applied_seq_id, - promotion_ctx->segments); - } - mooncake::RegisterRpcService(server, *wrapped_master_service); + mooncake::RegisterRpcService(server, *wrapped_master_service); - auto serve_preflight = - leader_coordinator.RenewLeadership(*leadership_session); - if (!serve_preflight) { - DeactivateServingState(admin_server, label_reconciler); - EnterStandbyMode(admin_server, *standby_controller, - accept_standby_runtime_updates, - leadership_session->view); - if (HandleLeadershipPhaseError( - "serve preflight failure", - "validate leadership before serving", leader_coordinator, - *leadership_session, serve_preflight.error(), spec.type)) { + async_simple::Future ec = server.async_start(); + if (ec.hasResult()) { + LOG(ERROR) << "Failed to start master service: " + << ec.result().value(); + DeactivateServingState(admin_server, label_reconciler); + EnterStandbyMode(admin_server, *standby_controller, + accept_standby_runtime_updates, + cvm_controller); return -1; } - continue; - } - if (!serve_preflight.value()) { - DeactivateServingState(admin_server, label_reconciler); - EnterStandbyMode(admin_server, *standby_controller, - accept_standby_runtime_updates, std::nullopt); - LogLeadershipReleaseWarning( - "serve preflight expiration", - leader_coordinator.ReleaseLeadership(*leadership_session)); - LOG(WARNING) << "Leadership expired before entering serve phase"; - admin_server.SetObservedLeader(std::nullopt); - continue; - } - std::atomic serve_shutdown_requested{false}; - auto leadership_monitor = leader_coordinator.StartLeadershipMonitor( - *leadership_session, - [&server, &admin_server, &serve_shutdown_requested, - &label_reconciler, &metrics_reporter](auto reason) { - metrics_reporter.Stop(); - serve_shutdown_requested.store(true, std::memory_order_release); - admin_server.SetServiceAvailable(false); - label_reconciler.SetLeader(false); - SetRuntimeState(admin_server, MasterRuntimeState::kStandby); - LOG(INFO) << "Trying to stop server, reason=" - << LeadershipLossReasonToString(reason); + // Lightweight demotion signal: only stops the coro_rpc server so + // the blocking get() below unblocks; the main loop then performs + // the full downgrade sequence serially. + cvm_membership_coordinator.SetServeStopSignal([&server]() { + LOG(INFO) << "CVM quota demotion: stopping server"; server.stop(); }); - if (!leadership_monitor) { - DeactivateServingState(admin_server, label_reconciler); - EnterStandbyMode(admin_server, *standby_controller, - accept_standby_runtime_updates, - leadership_session->view); - if (HandleLeadershipPhaseError( - "serve monitor startup failure", "start leadership monitor", - leader_coordinator, *leadership_session, - leadership_monitor.error(), spec.type)) { - return -1; - } - continue; - } - auto leadership_monitor_handle = std::move(leadership_monitor.value()); - async_simple::Future ec = server.async_start(); - if (ec.hasResult()) { - LOG(ERROR) << "Failed to start master service: " - << ec.result().value(); - StopLeadershipMonitor(leadership_monitor_handle); - DeactivateServingState(admin_server, label_reconciler); - EnterStandbyMode(admin_server, *standby_controller, - accept_standby_runtime_updates, - leadership_session->view); - auto err = - leader_coordinator.ReleaseLeadership(*leadership_session); - if (err != ErrorCode::OK) { - LOG(ERROR) << "Failed to release leadership: " << toString(err); + // A demotion may have been decided while the server was starting + // up (before the stop signal was installed). If our role is no + // longer primary, stop the server now so the get() below returns + // promptly instead of blocking forever. + const bool demoted_during_startup = + cvm_controller && + cvm_controller->GetCurrentRole() == cvm::MasterRole::kStandby; + + if (!demoted_during_startup) { + ActivateServingState(admin_server, wrapped_master_service, + label_reconciler); + metrics_reporter.SetRole("primary"); + metrics_reporter.Start(); + if (wrapped_master_service->StartSlotOwnerHeartbeat() != + ErrorCode::OK) { + LOG(WARNING) << "Failed to start SlotOwnerHeartbeat during " + "serve phase"; + } + // Inter-master RPC channel: etcd-driven peer discovery + + // cached peer connections for the CVM multi-submaster + // coordination (handshake now, allocation forwarding later). + if (wrapped_master_service->StartInterMasterRpc() != + ErrorCode::OK) { + LOG(WARNING) << "Failed to start InterMasterRpcClient " + "during serve phase"; + } + } else { + LOG(INFO) << "Demoted during serve startup; stopping server"; + server.stop(); } - return -1; - } - if (!serve_shutdown_requested.load(std::memory_order_acquire)) { - ActivateServingState(admin_server, wrapped_master_service, - label_reconciler); - metrics_reporter.SetRole("primary"); - metrics_reporter.Start(); - } + auto server_err = std::move(ec).get(); + LOG(INFO) << "Master service stopped: " << server_err; - auto server_err = std::move(ec).get(); - LOG(ERROR) << "Master service stopped: " << server_err; - - metrics_reporter.Stop(); - metrics_reporter.SetRole("standby"); - StopLeadershipMonitor(leadership_monitor_handle); - DeactivateServingState(admin_server, label_reconciler); - auto err = leader_coordinator.ReleaseLeadership(*leadership_session); - LOG(INFO) << "Release leadership: " << toString(err); - auto current_view = leader_coordinator.ReadCurrentView(); - if (current_view) { + // ── Downgrade sequence (kPrimary → kStandby) ── + metrics_reporter.Stop(); + metrics_reporter.SetRole("standby"); + wrapped_master_service->StopSlotOwnerHeartbeat(); + wrapped_master_service->StopInterMasterRpc(); + cvm_membership_coordinator.SetServeStopSignal(nullptr); + DeactivateServingState(admin_server, label_reconciler); EnterStandbyMode(admin_server, *standby_controller, accept_standby_runtime_updates, - current_view.value()); + cvm_controller); } else { + // ── Standby (kStandby): keep replicating, wait for promotion ── EnterStandbyMode(admin_server, *standby_controller, - accept_standby_runtime_updates, std::nullopt); + accept_standby_runtime_updates, + cvm_controller); + if (cvm_controller) { + cvm_membership_coordinator.WaitForRoleOrViewChange(); + } else { + // Unreachable: target_role is kPrimary when cvm_controller is + // null. Sleep defensively to avoid a tight loop. + std::this_thread::sleep_for(kSupervisorRetryInterval); + } } } diff --git a/mooncake-store/src/ha/oplog/oplog_batch_standby_reader.cpp b/mooncake-store/src/ha/oplog/oplog_batch_standby_reader.cpp index 1c1d9295bd..c7794c6540 100644 --- a/mooncake-store/src/ha/oplog/oplog_batch_standby_reader.cpp +++ b/mooncake-store/src/ha/oplog/oplog_batch_standby_reader.cpp @@ -26,8 +26,10 @@ void SetPollError(OpLogBatchStandbyPollResult& result, ErrorCode error, OpLogBatchStandbyReader::OpLogBatchStandbyReader(std::string cluster_id, HaKvBackend& backend, - OpLogApplier& applier) - : storage_(std::move(cluster_id), backend), applier_(applier) {} + OpLogApplier& applier, + std::string source_id) + : storage_(std::move(cluster_id), backend, std::move(source_id)), + applier_(applier) {} OpLogBatchStandbyPollResult OpLogBatchStandbyReader::PollOnce( size_t max_batches) { diff --git a/mooncake-store/src/ha/oplog/oplog_batch_storage.cpp b/mooncake-store/src/ha/oplog/oplog_batch_storage.cpp index be2a66c964..f38175fac5 100644 --- a/mooncake-store/src/ha/oplog/oplog_batch_storage.cpp +++ b/mooncake-store/src/ha/oplog/oplog_batch_storage.cpp @@ -2,6 +2,7 @@ #include #include +#include #include @@ -53,8 +54,11 @@ bool TryParseBatchIdFromKey(const std::string& key, uint64_t& batch_id) { } // namespace OpLogBatchStorage::OpLogBatchStorage(std::string cluster_id, - HaKvBackend& backend) - : cluster_id_(std::move(cluster_id)), backend_(backend) { + HaKvBackend& backend, + std::string source_id) + : cluster_id_(std::move(cluster_id)), + source_id_(std::move(source_id)), + backend_(backend) { cluster_id_valid_ = NormalizeAndValidateClusterId(cluster_id_) && !cluster_id_.empty(); } @@ -78,7 +82,7 @@ ErrorCode OpLogBatchStorage::InitDurablePrefix(DurablePrefix& prefix) { return ErrorCode::INVALID_PARAMS; } - auto batch_range = BuildBatchRecordRange(cluster_id_, 0); + auto batch_range = BuildBatchRecordRange(cluster_id_, 0, source_id_); std::vector existing_batches; err = backend_.Range(batch_range.begin_key, batch_range.end_key, /*limit=*/1, existing_batches); @@ -90,7 +94,7 @@ ErrorCode OpLogBatchStorage::InitDurablePrefix(DurablePrefix& prefix) { return ErrorCode::INTERNAL_ERROR; } - const std::string durable_key = BuildDurablePrefixKey(cluster_id_); + const std::string durable_key = BuildDurablePrefixKey(cluster_id_, source_id_); DurablePrefix initial{.batch_id = 0, .last_seq = 0}; KvTxn txn; txn.compares.push_back({.key = durable_key, @@ -114,7 +118,7 @@ ErrorCode OpLogBatchStorage::ReadDurablePrefix(DurablePrefix& prefix) { return ErrorCode::INVALID_PARAMS; } std::string value; - const std::string key = BuildDurablePrefixKey(cluster_id_); + const std::string key = BuildDurablePrefixKey(cluster_id_, source_id_); ErrorCode err = backend_.Get(key, value); if (err != ErrorCode::OK) { return err; @@ -157,7 +161,7 @@ ErrorCode OpLogBatchStorage::WriteBatchAndAdvancePrefix( return ErrorCode::INVALID_PARAMS; } - const std::string durable_key = BuildDurablePrefixKey(cluster_id_); + const std::string durable_key = BuildDurablePrefixKey(cluster_id_, source_id_); const std::string encoded_batch = EncodeOpLogBatchRecord(batch); #ifdef MOONCAKE_ENABLE_OPLOG_PERF_METRICS HAMetricManager::instance().observe_batch_record_batch_bytes( @@ -168,8 +172,9 @@ ErrorCode OpLogBatchStorage::WriteBatchAndAdvancePrefix( {.key = durable_key, .kind = KvCompareKind::kValueEquals, .expected_value = EncodeDurablePrefix(expected_prefix)}); - txn.puts.push_back({.key = BuildBatchRecordKey(cluster_id_, batch.batch_id), - .value = encoded_batch}); + txn.puts.push_back( + {.key = BuildBatchRecordKey(cluster_id_, batch.batch_id, source_id_), + .value = encoded_batch}); txn.puts.push_back( {.key = durable_key, .value = EncodeDurablePrefix( @@ -200,7 +205,8 @@ ErrorCode OpLogBatchStorage::ReadBatch(uint64_t batch_id, } std::string value; ErrorCode err = - backend_.Get(BuildBatchRecordKey(cluster_id_, batch_id), value); + backend_.Get(BuildBatchRecordKey(cluster_id_, batch_id, source_id_), + value); if (err != ErrorCode::OK) { return err; } @@ -219,7 +225,7 @@ ErrorCode OpLogBatchStorage::ReadBatchesAfter( if (!IsValidClusterId()) { return ErrorCode::INVALID_PARAMS; } - auto range = BuildBatchRecordRange(cluster_id_, after_batch_id); + auto range = BuildBatchRecordRange(cluster_id_, after_batch_id, source_id_); std::string begin_key = range.begin_key; do { std::vector kvs; @@ -273,27 +279,60 @@ ErrorCode OpLogBatchStorage::RejectLegacyLayout() const { return err; } - std::vector entries; - err = backend_.Range(root + "00000000000000000000", root + ":", - /*limit=*/1, entries); - if (err != ErrorCode::OK) { - return err; - } - if (!entries.empty()) { - LOG(ERROR) << "Legacy per-entry OpLog key exists for cluster=" - << cluster_id_ - << "; clear the legacy OpLog namespace before enabling " - "batch-record OpLog"; - return ErrorCode::INCOMPLETE_OPLOG_CATCH_UP; - } + // Legacy per-entry keys have the exact shape root + <20 digits> with no + // trailing '/'. Batch-layout keys (root + /batches/... and + // root + /durable_prefix) can sort lexically inside the legacy + // numeric range when the source id starts with a digit (e.g. an + // "ip:port" master id such as "141.61.84.245:50052"), so a bare range + // probe would false-positive on this node's own batch keys after a + // restart. Scan the range with pagination and reject only keys matching + // the legacy per-entry shape. + std::string begin_key = root + "00000000000000000000"; + const std::string end_key = root + ":"; + constexpr size_t kLegacyScanPageLimit = 100; + do { + std::vector entries; + err = backend_.Range(begin_key, end_key, kLegacyScanPageLimit, entries); + if (err != ErrorCode::OK) { + return err; + } + for (const auto& kv : entries) { + const std::string_view suffix(kv.key.data() + root.size(), + kv.key.size() - root.size()); + bool is_legacy_entry = suffix.size() == + static_cast( + kOpLogBatchIdWidth) && + suffix.find('/') == + std::string_view::npos; + if (is_legacy_entry) { + for (char c : suffix) { + if (c < '0' || c > '9') { + is_legacy_entry = false; + break; + } + } + } + if (is_legacy_entry) { + LOG(ERROR) << "Legacy per-entry OpLog key exists for cluster=" + << cluster_id_ << ": key=" << kv.key + << "; clear the legacy OpLog namespace before " + "enabling batch-record OpLog"; + return ErrorCode::INCOMPLETE_OPLOG_CATCH_UP; + } + } + if (entries.size() < kLegacyScanPageLimit) { + break; + } + begin_key = entries.back().key + '\0'; + } while (begin_key < end_key); - entries.clear(); + std::vector snapshot_entries; err = backend_.Range(root + "snapshot/", root + "snapshot0", - /*limit=*/1, entries); + /*limit=*/1, snapshot_entries); if (err != ErrorCode::OK) { return err; } - if (!entries.empty()) { + if (!snapshot_entries.empty()) { LOG(ERROR) << "Legacy OpLog snapshot sidecar exists for cluster=" << cluster_id_ << "; clear the legacy OpLog namespace before enabling " diff --git a/mooncake-store/src/ha/oplog/oplog_batch_types.cpp b/mooncake-store/src/ha/oplog/oplog_batch_types.cpp index 2e91730d31..66cf25f813 100644 --- a/mooncake-store/src/ha/oplog/oplog_batch_types.cpp +++ b/mooncake-store/src/ha/oplog/oplog_batch_types.cpp @@ -25,8 +25,22 @@ std::string PrefixEnd(std::string prefix) { return std::string(1, '\0'); } -std::string BatchPrefix(const std::string& cluster_id) { - return "/oplog/" + cluster_id + "/batches/"; +std::string BatchPrefix(const std::string& cluster_id, + const std::string& source_id) { + std::string path = "/oplog/" + cluster_id + "/"; + if (!source_id.empty()) { + path += source_id + "/"; + } + return path + "batches/"; +} + +std::string DurablePrefixPath(const std::string& cluster_id, + const std::string& source_id) { + std::string path = "/oplog/" + cluster_id + "/"; + if (!source_id.empty()) { + path += source_id + "/"; + } + return path + "durable_prefix"; } } // namespace @@ -114,29 +128,32 @@ std::string FormatOpLogBatchId(uint64_t batch_id) { } std::string BuildBatchRecordKey(const std::string& cluster_id, - uint64_t batch_id) { + uint64_t batch_id, + const std::string& source_id) { std::string normalized = cluster_id; if (!NormalizeAndValidateClusterId(normalized) || normalized.empty()) { return {}; } - return BatchPrefix(normalized) + FormatOpLogBatchId(batch_id); + return BatchPrefix(normalized, source_id) + FormatOpLogBatchId(batch_id); } -std::string BuildDurablePrefixKey(const std::string& cluster_id) { +std::string BuildDurablePrefixKey(const std::string& cluster_id, + const std::string& source_id) { std::string normalized = cluster_id; if (!NormalizeAndValidateClusterId(normalized) || normalized.empty()) { return {}; } - return "/oplog/" + normalized + "/durable_prefix"; + return DurablePrefixPath(normalized, source_id); } BatchRecordRange BuildBatchRecordRange(const std::string& cluster_id, - uint64_t after_batch_id) { + uint64_t after_batch_id, + const std::string& source_id) { std::string normalized = cluster_id; if (!NormalizeAndValidateClusterId(normalized) || normalized.empty()) { return {}; } - const std::string prefix = BatchPrefix(normalized); + const std::string prefix = BatchPrefix(normalized, source_id); if (after_batch_id == UINT64_MAX) { return {.begin_key = PrefixEnd(prefix), .end_key = PrefixEnd(prefix)}; } diff --git a/mooncake-store/src/ha/standby_controller.cpp b/mooncake-store/src/ha/standby_controller.cpp index 6576cd9a21..08ae18da0f 100644 --- a/mooncake-store/src/ha/standby_controller.cpp +++ b/mooncake-store/src/ha/standby_controller.cpp @@ -47,7 +47,7 @@ StandbyRuntimeCapabilities BuildStandbyRuntimeCapabilities( MasterRuntimeState MapStandbyRuntimeState( const StandbySyncStatus& status, - const std::optional& observed_leader, + const MasterSources& sources, const StandbyRuntimeCapabilities& capabilities) { switch (status.state) { case StandbyState::STOPPED: @@ -60,7 +60,7 @@ MasterRuntimeState MapStandbyRuntimeState( return MasterRuntimeState::kRecovering; case StandbyState::WATCHING: if (capabilities.has_oplog_following && - observed_leader.has_value() && status.lag_entries > 0) { + !sources.empty() && status.lag_entries > 0) { return MasterRuntimeState::kCatchingUp; } return MasterRuntimeState::kStandby; @@ -73,7 +73,7 @@ MasterRuntimeState MapStandbyRuntimeState( class NoopStandbyController final : public StandbyController { public: - ErrorCode StartStandby(const std::optional&) override { + ErrorCode StartStandby(const MasterSources&) override { return ErrorCode::OK; } @@ -83,10 +83,12 @@ class NoopStandbyController final : public StandbyController { tl::expected PromoteStandbyAndExport() override { - return tl::unexpected(ErrorCode::UNAVAILABLE_IN_CURRENT_STATUS); + // 无 standby 回放能力时没有数据可导出，返回空上下文让 primary 直接 + // serving（数据复制/迁移由 P2/P4 处理），而非让升级序列死循环。 + return PromotionContext{}; } - void UpdateObservedLeader(const std::optional&) override {} + void UpdateObservedLeader(const MasterSources&) override {} MasterRuntimeState GetStandbyRuntimeState() const override { return MasterRuntimeState::kStandby; @@ -149,17 +151,22 @@ class CapabilityDrivenStandbyController final : public StandbyController { standby_service_->Stop(); } - ErrorCode StartStandby( - const std::optional& observed_leader) override { + ErrorCode StartStandby(const MasterSources& sources) override { bool standby_running = false; { std::lock_guard lock(state_mutex_); - observed_leader_ = observed_leader; + observed_sources_ = sources; standby_running = standby_running_; } if (standby_running) { - NotifyRuntimeStateIfChanged(); - return ErrorCode::OK; + // 运行中：动态重绑（若源集合变化）。UpdateSources 内部幂等，未 + // 运行则等价 Start，源集合不变则 no-op，变化则 Stop+Start。 + ErrorCode err = standby_service_->UpdateSources( + sources, oplog_connstring_, config_.cluster_id); + if (err == ErrorCode::OK) { + NotifyRuntimeStateIfChanged(); + } + return err; } if (dependency_init_error_ != ErrorCode::OK) { @@ -169,8 +176,7 @@ class CapabilityDrivenStandbyController final : public StandbyController { } ErrorCode err = standby_service_->Start( - observed_leader.has_value() ? observed_leader->leader_address : "", - oplog_connstring_, config_.cluster_id); + sources, oplog_connstring_, config_.cluster_id); { std::lock_guard lock(state_mutex_); @@ -274,28 +280,27 @@ class CapabilityDrivenStandbyController final : public StandbyController { return ctx; } - void UpdateObservedLeader( - const std::optional& observed_leader) override { + void UpdateObservedLeader(const MasterSources& sources) override { { std::lock_guard lock(state_mutex_); - observed_leader_ = observed_leader; + observed_sources_ = sources; } NotifyRuntimeStateIfChanged(); } MasterRuntimeState GetStandbyRuntimeState() const override { - std::optional observed_leader; + MasterSources observed_sources; bool standby_running = false; { std::lock_guard lock(state_mutex_); - observed_leader = observed_leader_; + observed_sources = observed_sources_; standby_running = standby_running_; } if (!standby_running) { return MasterRuntimeState::kStandby; } return MapStandbyRuntimeState(standby_service_->GetSyncStatus(), - observed_leader, capabilities_); + observed_sources, capabilities_); } void SetStandbyRuntimeStateCallback( @@ -338,7 +343,7 @@ class CapabilityDrivenStandbyController final : public StandbyController { std::string oplog_connstring_; mutable std::mutex state_mutex_; - std::optional observed_leader_; + MasterSources observed_sources_; bool standby_running_ = false; ErrorCode last_standby_error_{ErrorCode::OK}; diff --git a/mooncake-store/src/hot_standby_service.cpp b/mooncake-store/src/hot_standby_service.cpp index c31ae1e2e4..fa91f24822 100644 --- a/mooncake-store/src/hot_standby_service.cpp +++ b/mooncake-store/src/hot_standby_service.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include "etcd_helper.h" @@ -19,8 +20,8 @@ namespace mooncake { HotStandbyService::HotStandbyService(const HotStandbyConfig& config) : config_(config) { metadata_store_ = std::make_unique(); - // OpLogApplier is re-created in Start() with the resolved cluster_id. - oplog_applier_ = std::make_unique(metadata_store_.get()); + // Per-source OpLogApplier/reader replicas are created in Start() with the + // resolved cluster_id and source list. // Register callback for state change logging and metrics. state_machine_.RegisterCallback([this](StandbyState old_state, @@ -153,7 +154,7 @@ HotStandbyService::~HotStandbyService() { } } -ErrorCode HotStandbyService::Start(const std::string& primary_address, +ErrorCode HotStandbyService::Start(const std::vector& sources, const std::string& oplog_endpoints, const std::string& cluster_id) { std::lock_guard lock(mutex_); @@ -174,8 +175,9 @@ ErrorCode HotStandbyService::Start(const std::string& primary_address, if (verification_thread_.joinable()) { verification_thread_.join(); } - batch_standby_reader_.reset(); + sources_.clear(); batch_standby_kv_backend_.reset(); + baseline_segments_.clear(); last_error_.store(ErrorCode::OK, std::memory_order_release); @@ -186,7 +188,7 @@ ErrorCode HotStandbyService::Start(const std::string& primary_address, return ErrorCode::INTERNAL_ERROR; // State machine rejected START } - config_.primary_address = primary_address; + config_.sources = sources; oplog_endpoints_ = oplog_endpoints; cluster_id_ = cluster_id; @@ -226,13 +228,41 @@ ErrorCode HotStandbyService::Start(const std::string& primary_address, } uint64_t HotStandbyService::GetLocalLastAppliedSequenceIdLocked() const { - if (oplog_applier_) { - uint64_t expected = oplog_applier_->GetExpectedSequenceId(); - return expected > 0 ? expected - 1 : 0; + uint64_t max_applied = 0; + for (const auto& [source_id, replica] : sources_) { + if (replica.applier) { + const uint64_t expected = replica.applier->GetExpectedSequenceId(); + const uint64_t applied = expected > 0 ? expected - 1 : 0; + max_applied = std::max(max_applied, applied); + } + } + if (max_applied > 0) { + return max_applied; } return applied_seq_id_.load(std::memory_order_acquire); } +void HotStandbyService::CollectSegmentsLocked( + std::vector& out) const { + out.clear(); + // Merge segment registries across per-source appliers, deduplicating by + // transport_endpoint (segment identity in StandbySegmentRegistry). + std::unordered_map by_endpoint; + for (const auto& [source_id, replica] : sources_) { + if (!replica.applier) { + continue; + } + for (const auto& segment : + replica.applier->GetSegmentRegistry().GetAllSegments()) { + by_endpoint[segment.transport_endpoint] = segment; + } + } + out.reserve(by_endpoint.size()); + for (const auto& [endpoint, segment] : by_endpoint) { + out.push_back(segment); + } +} + ErrorCode HotStandbyService::PrepareBootstrapBaselineLocked( uint64_t& baseline_seq_id) { const uint64_t local_last_seq_id = GetLocalLastAppliedSequenceIdLocked(); @@ -248,8 +278,6 @@ ErrorCode HotStandbyService::PrepareBootstrapBaselineLocked( metadata_store_ = std::make_unique(); } - oplog_applier_ = - std::make_unique(metadata_store_.get(), cluster_id_); if (!config_.enable_oplog_following) { if (metadata_store_ && metadata_store_->GetKeyCount() > 0) { LOG(INFO) << "Snapshot-only restart discards local metadata and " @@ -269,7 +297,6 @@ ErrorCode HotStandbyService::PrepareBootstrapBaselineLocked( LOG(INFO) << "Standby warm start: reuse local metadata (keys=" << metadata_store_->GetKeyCount() << "), recover last_seq_id=" << local_last_seq_id; - oplog_applier_->Recover(local_last_seq_id); baseline_seq_id = local_last_seq_id; } else { auto snapshot_err = LoadSnapshotBaselineLocked(baseline_seq_id); @@ -287,7 +314,7 @@ ErrorCode HotStandbyService::LoadSnapshotBaselineLocked( uint64_t& baseline_seq_id) { baseline_seq_id = 0; metadata_store_->Clear(); - oplog_applier_->Recover(0); + baseline_segments_.clear(); if (!config_.enable_snapshot_bootstrap || !snapshot_provider_) { return ErrorCode::OK; @@ -327,11 +354,9 @@ ErrorCode HotStandbyService::LoadSnapshotBaselineLocked( metadata_store_->PutMetadata(entry.tenant_id, entry.key, entry.metadata); } - // Load segment registry from snapshot - if (oplog_applier_) { - oplog_applier_->LoadSegmentRegistry(snapshot.segments); - } - oplog_applier_->Recover(snapshot.snapshot_sequence_id); + // Load segment registry from snapshot; applied to each per-source applier + // when OpLog following starts. + baseline_segments_ = snapshot.segments; baseline_seq_id = snapshot.snapshot_sequence_id; return ErrorCode::OK; } @@ -344,8 +369,18 @@ ErrorCode HotStandbyService::StartOplogFollowingLocked( } else { batch_standby_kv_backend_ = std::make_shared(); } - batch_standby_reader_ = std::make_unique( - cluster_id_, *batch_standby_kv_backend_, *oplog_applier_); + + sources_.clear(); + for (const auto& source : config_.sources) { + auto applier = + std::make_unique(metadata_store_.get(), cluster_id_); + applier->LoadSegmentRegistry(baseline_segments_); + auto reader = std::make_unique( + cluster_id_, *batch_standby_kv_backend_, *applier, source.master_id); + sources_.emplace( + source.master_id, + SourceReplica{std::move(applier), std::move(reader)}); + } state_machine_.ProcessEvent(StandbyEvent::SYNC_COMPLETE); replication_loop_running_.store(true, std::memory_order_release); @@ -357,7 +392,8 @@ ErrorCode HotStandbyService::StartOplogFollowingLocked( } LOG(INFO) << "HotStandbyService started, watching OpLog for cluster: " - << cluster_id_ << ", state=" << StandbyStateToString(GetState()); + << cluster_id_ << ", sources=" << sources_.size() + << ", state=" << StandbyStateToString(GetState()); return ErrorCode::OK; } @@ -423,19 +459,62 @@ void HotStandbyService::Stop() { << StandbyStateToString(GetState()); } +ErrorCode HotStandbyService::UpdateSources( + const std::vector& sources, + const std::string& oplog_endpoints, + const std::string& cluster_id) { + // 判断是否需要重绑（比较 master_id 集合）。 + bool running = false; + bool changed = false; + { + std::lock_guard lock(mutex_); + running = IsRunning(); + if (running) { + if (config_.sources.size() != sources.size()) { + changed = true; + } else { + std::set old_ids; + std::set new_ids; + for (const auto& s : config_.sources) { + old_ids.insert(s.master_id); + } + for (const auto& s : sources) { + new_ids.insert(s.master_id); + } + changed = (old_ids != new_ids); + } + } + } + + if (!running) { + return Start(sources, oplog_endpoints, cluster_id); + } + if (!changed) { + return ErrorCode::OK; + } + + // 运行中且源集合变化：停止后重启。metadata_store_ 是成员变量，Stop 不 + // 清空，Start 里 PrepareBootstrapBaselineLocked 会复用已回放的元数据。 + LOG(INFO) << "HotStandbyService rebinding replay sources: " + << config_.sources.size() << " -> " << sources.size(); + Stop(); + return Start(sources, oplog_endpoints, cluster_id); +} + StandbySyncStatus HotStandbyService::GetSyncStatus() const { StandbySyncStatus status; - // Get applied sequence ID from OpLogApplier - if (oplog_applier_) { - uint64_t expected = oplog_applier_->GetExpectedSequenceId(); - status.applied_seq_id = (expected > 0) ? (expected - 1) : 0; - if (status.applied_seq_id == 0) { - status.applied_seq_id = applied_seq_id_.load(); // Fallback + // Get applied sequence ID from per-source OpLogAppliers (max across + // sources; each source owns an independent sequence space). + uint64_t max_applied = 0; + for (const auto& [source_id, replica] : sources_) { + if (replica.applier) { + const uint64_t expected = replica.applier->GetExpectedSequenceId(); + max_applied = std::max(max_applied, expected > 0 ? expected - 1 : 0); } - } else { - status.applied_seq_id = applied_seq_id_.load(); } + status.applied_seq_id = + max_applied > 0 ? max_applied : applied_seq_id_.load(); // Primary sequence ID (best-effort): updated by ReplicationLoop via etcd // `/latest`. @@ -494,9 +573,9 @@ ErrorCode HotStandbyService::FinalCatchUpForPromotionLocked( LOG(INFO) << "Promotion does not require final OpLog catch-up"; return ErrorCode::OK; } - if (!oplog_applier_) { - LOG(ERROR) << "Final catch-up requires OpLogApplier"; - return ErrorCode::INTERNAL_ERROR; + if (sources_.empty()) { + LOG(INFO) << "Final catch-up skipped: no OpLog sources"; + return ErrorCode::OK; } if (catch_up_batch_kv_backend_for_testing_) { @@ -510,55 +589,60 @@ ErrorCode HotStandbyService::FinalCatchUpForPromotionLocked( ErrorCode HotStandbyService::FinalCatchUpBatchRecordsLocked( HaKvBackend& backend) { - std::unique_ptr local_reader; - OpLogBatchStandbyReader* reader = batch_standby_reader_.get(); - if (reader == nullptr) { - local_reader = std::make_unique( - cluster_id_, backend, *oplog_applier_); - reader = local_reader.get(); - } const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(30); const auto initial_retry_delay = std::chrono::milliseconds(std::max(config_.oplog_poll_interval_ms, 1)); const auto max_retry_delay = std::max(initial_retry_delay, std::chrono::milliseconds(1000)); - auto retry_delay = initial_retry_delay; - auto wait_to_retry = [&] { - const auto now = std::chrono::steady_clock::now(); - if (now >= deadline) { - return false; - } - std::this_thread::sleep_for(std::min( - retry_delay, std::chrono::duration_cast( - deadline - now))); - retry_delay = std::min(retry_delay * 2, max_retry_delay); - return true; - }; - for (;;) { - if (std::chrono::steady_clock::now() >= deadline) { - return ErrorCode::INCOMPLETE_OPLOG_CATCH_UP; + + for (auto& [source_id, replica] : sources_) { + OpLogBatchStandbyReader* reader = replica.reader.get(); + std::unique_ptr local_reader; + if (reader == nullptr) { + local_reader = std::make_unique( + cluster_id_, backend, *replica.applier, source_id); + reader = local_reader.get(); } - auto result = reader->PollOnce(); - if (result.error != ErrorCode::OK) { - if (result.disposition != - OpLogBatchStandbyPollDisposition::RETRYABLE || - !wait_to_retry()) { + + auto retry_delay = initial_retry_delay; + auto wait_to_retry = [&] { + const auto now = std::chrono::steady_clock::now(); + if (now >= deadline) { + return false; + } + std::this_thread::sleep_for(std::min( + retry_delay, + std::chrono::duration_cast( + deadline - now))); + retry_delay = std::min(retry_delay * 2, max_retry_delay); + return true; + }; + for (;;) { + if (std::chrono::steady_clock::now() >= deadline) { return ErrorCode::INCOMPLETE_OPLOG_CATCH_UP; } - continue; - } - retry_delay = initial_retry_delay; - if (!result.durable_prefix_present) { - return GetLocalLastAppliedSequenceIdLocked() == 0 - ? ErrorCode::OK - : ErrorCode::INCOMPLETE_OPLOG_CATCH_UP; - } - if (GetLocalLastAppliedSequenceIdLocked() >= - result.durable_prefix.last_seq) { - return ErrorCode::OK; + auto result = reader->PollOnce(); + if (result.error != ErrorCode::OK) { + if (result.disposition != + OpLogBatchStandbyPollDisposition::RETRYABLE || + !wait_to_retry()) { + return ErrorCode::INCOMPLETE_OPLOG_CATCH_UP; + } + continue; + } + retry_delay = initial_retry_delay; + if (!result.durable_prefix_present) { + break; // Source has no OpLog yet. + } + const uint64_t expected = replica.applier->GetExpectedSequenceId(); + const uint64_t source_applied = expected > 0 ? expected - 1 : 0; + if (source_applied >= result.durable_prefix.last_seq) { + break; // This source is caught up. + } } } + return ErrorCode::OK; } ErrorCode HotStandbyService::Promote() { @@ -661,11 +745,7 @@ ErrorCode HotStandbyService::PromoteAndExportSnapshot(StandbySnapshot& out) { } else { out.objects.clear(); } - if (oplog_applier_) { - out.segments = oplog_applier_->GetSegmentRegistry().GetAllSegments(); - } else { - out.segments.clear(); - } + CollectSegmentsLocked(out.segments); lock.unlock(); Stop(); @@ -686,13 +766,7 @@ size_t HotStandbyService::GetMetadataCount() const { uint64_t HotStandbyService::GetLatestAppliedSequenceId() const { std::lock_guard lock(mutex_); - if (oplog_applier_) { - uint64_t expected_seq = oplog_applier_->GetExpectedSequenceId(); - // GetExpectedSequenceId returns the next expected sequence_id, - // so the latest applied is expected_seq - 1 - return expected_seq > 0 ? expected_seq - 1 : 0; - } - return applied_seq_id_.load(); + return GetLocalLastAppliedSequenceIdLocked(); } bool HotStandbyService::ExportMetadataSnapshot( @@ -712,13 +786,8 @@ bool HotStandbyService::ExportStandbySnapshot(StandbySnapshot& out) const { return false; } - // Get applied sequence ID (inline to avoid recursive mutex lock) - if (oplog_applier_) { - uint64_t expected_seq = oplog_applier_->GetExpectedSequenceId(); - out.oplog_sequence_id = expected_seq > 0 ? expected_seq - 1 : 0; - } else { - out.oplog_sequence_id = applied_seq_id_.load(); - } + // Get applied sequence ID (max across per-source appliers) + out.oplog_sequence_id = GetLocalLastAppliedSequenceIdLocked(); // Export object metadata if (metadata_store_) { @@ -727,12 +796,8 @@ bool HotStandbyService::ExportStandbySnapshot(StandbySnapshot& out) const { out.objects.clear(); } - // Export segments from OpLogApplier's registry (Patch B) - if (oplog_applier_) { - out.segments = oplog_applier_->GetSegmentRegistry().GetAllSegments(); - } else { - out.segments.clear(); - } + // Export segments merged across per-source appliers + CollectSegmentsLocked(out.segments); return true; } @@ -775,10 +840,19 @@ void HotStandbyService::ReplicationLoop() { continue; } - if (batch_standby_reader_) { + // Poll each replay source independently. Sources own disjoint slot + // ranges and independent OpLog sequence spaces, so aggregate applied + // and primary sequence IDs as the max across all sources. + bool retry_backoff = false; + bool fatal = false; + for (auto& [source_id, replica] : sources_) { + if (!replica.reader) { + continue; + } + const uint64_t expected_before = - oplog_applier_->GetExpectedSequenceId(); - auto result = batch_standby_reader_->PollOnce(); + replica.applier->GetExpectedSequenceId(); + auto result = replica.reader->PollOnce(); if (result.durable_prefix_present) { const uint64_t current_primary = primary_seq_id_.load(); if (result.durable_prefix.last_seq > current_primary) { @@ -787,10 +861,14 @@ void HotStandbyService::ReplicationLoop() { } const uint64_t expected_after = - oplog_applier_->GetExpectedSequenceId(); + replica.applier->GetExpectedSequenceId(); if (expected_after > 0) { - applied_seq_id_.store(expected_after - 1); + const uint64_t applied = expected_after - 1; + if (applied > applied_seq_id_.load()) { + applied_seq_id_.store(applied); + } } + if (result.error != ErrorCode::OK) { last_error_.store(result.error, std::memory_order_release); const bool made_progress = expected_after > expected_before; @@ -803,39 +881,43 @@ void HotStandbyService::ReplicationLoop() { } if (now - *retry_started < retry_limit) { LOG(WARNING) - << "Transient batch-record standby poll failure, " - << "retrying in " << retry_delay.count() + << "Transient batch-record standby poll failure " + << "for source=" << source_id << ", retrying in " + << retry_delay.count() << " ms, err=" << static_cast(result.error); NotifySyncStatus(); wait_for_next_poll(retry_delay); retry_delay = std::min(retry_delay * 2, max_retry_delay); - continue; + retry_backoff = true; + break; } LOG(ERROR) << "Batch-record standby retry timeout after " << config_.batch_oplog_retry_timeout_sec - << " seconds, err=" << static_cast(result.error); + << " seconds for source=" << source_id + << ", err=" << static_cast(result.error); } else { - LOG(ERROR) << "Fatal batch-record standby poll failure, " - << "err=" << static_cast(result.error); + LOG(ERROR) << "Fatal batch-record standby poll failure " + << "for source=" << source_id + << ", err=" << static_cast(result.error); } - state_machine_.ProcessEvent(StandbyEvent::FATAL_ERROR); - replication_loop_cv_.notify_all(); + fatal = true; break; } + retry_started.reset(); retry_delay = retry_base; last_error_.store(ErrorCode::OK, std::memory_order_release); } - // Update applied_seq_id from OpLogApplier - if (oplog_applier_) { - uint64_t expected = oplog_applier_->GetExpectedSequenceId(); - uint64_t current_applied = (expected > 0) ? (expected - 1) : 0; - if (current_applied > 0) { - applied_seq_id_.store(current_applied); - } + if (retry_backoff) { + continue; + } + if (fatal) { + state_machine_.ProcessEvent(StandbyEvent::FATAL_ERROR); + replication_loop_cv_.notify_all(); + break; } const uint64_t applied_seq_id = applied_seq_id_.load(); diff --git a/mooncake-store/src/master.cpp b/mooncake-store/src/master.cpp index 8ce9aadc55..b6176b11e7 100644 --- a/mooncake-store/src/master.cpp +++ b/mooncake-store/src/master.cpp @@ -297,6 +297,15 @@ DEFINE_int64(global_file_segment_size, DEFINE_string(cluster_id, mooncake::DEFAULT_CLUSTER_ID, "Cluster ID for the master service, used for kvcache persistence " "in HA mode"); +DEFINE_uint32(submaster_count, 1, + "Maximum number of simultaneously serving submaster in the " + "cluster (CVM quota coordination, first-come-first-served). " + "Masters ranked beyond this limit are demoted to standby."); +DEFINE_uint32(cvm_http_port, 0, + "Port for the CVM external HTTP API (CvmHttpServer). 0 keeps it " + "disabled."); +DEFINE_string(cvm_http_host, "0.0.0.0", + "Bind host for the CVM external HTTP API (CvmHttpServer)."); // OpLog store configuration DEFINE_bool(enable_oplog, false, @@ -460,6 +469,27 @@ void InitMasterConf(const mooncake::DefaultConfig& default_config, // Initialize the master service configuration from the default config default_config.GetBool("enable_cxl", &master_config.enable_cxl, FLAGS_enable_cxl); + default_config.GetBool("enable_vchunk", + &master_config.vchunk_config.enabled, false); + default_config.GetString("vchunk_etcd_endpoints", + &master_config.vchunk_etcd_endpoints, ""); + default_config.GetUInt64( + "vchunk_creating_timeout_ms", + &master_config.vchunk_config.creating_timeout_ms, 30'000); + default_config.GetUInt64( + "vchunk_releasing_timeout_ms", + &master_config.vchunk_config.releasing_timeout_ms, 60'000); + default_config.GetUInt32("vchunk_max_slice_retry", + &master_config.vchunk_config.max_slice_retry, 3); + default_config.GetUInt32("vchunk_max_slice_count", + &master_config.vchunk_config.max_slice_count, + 4096); + default_config.GetUInt64("vchunk_max_metadata_bytes", + &master_config.vchunk_config.max_metadata_bytes, + 1024U * 1024U); + default_config.GetUInt32( + "vchunk_max_creating_objects", + &master_config.vchunk_config.max_creating_objects, 1024); default_config.GetString("cxl_path", &master_config.cxl_path, FLAGS_cxl_path); default_config.GetUInt64("cxl_size", &master_config.cxl_size, @@ -1055,6 +1085,21 @@ void LoadConfigFromCmdline(mooncake::MasterConfig& master_config, !conf_set) { master_config.cluster_id = FLAGS_cluster_id; } + if ((google::GetCommandLineFlagInfo("submaster_count", &info) && + !info.is_default) || + !conf_set) { + master_config.submaster_count = FLAGS_submaster_count; + } + if ((google::GetCommandLineFlagInfo("cvm_http_port", &info) && + !info.is_default) || + !conf_set) { + master_config.cvm_http_port = static_cast(FLAGS_cvm_http_port); + } + if ((google::GetCommandLineFlagInfo("cvm_http_host", &info) && + !info.is_default) || + !conf_set) { + master_config.cvm_http_host = FLAGS_cvm_http_host; + } if ((google::GetCommandLineFlagInfo("enable_oplog", &info) && !info.is_default) || !conf_set) { diff --git a/mooncake-store/src/master_client.cpp b/mooncake-store/src/master_client.cpp index 7cc1b28c28..04896aa3c9 100644 --- a/mooncake-store/src/master_client.cpp +++ b/mooncake-store/src/master_client.cpp @@ -16,6 +16,8 @@ #include "mutex.h" #include "rpc_service.h" #include "types.h" +#include "etcd_helper.h" +#include "partition/kv_hash_map.h" #include "utils/scoped_vlog_timer.h" #include "master_metric_manager.h" #include "version.h" @@ -90,6 +92,35 @@ struct RpcNameTraits<&WrappedMasterService::PutRevoke> { static constexpr const char* value = "PutRevoke"; }; +template <> +struct RpcNameTraits<&WrappedMasterService::VChunkPutStart> { + static constexpr const char* value = "VChunkPutStart"; +}; +template <> +struct RpcNameTraits<&WrappedMasterService::VChunkPutEnd> { + static constexpr const char* value = "VChunkPutEnd"; +}; +template <> +struct RpcNameTraits<&WrappedMasterService::VChunkPutRevoke> { + static constexpr const char* value = "VChunkPutRevoke"; +}; +template <> +struct RpcNameTraits<&WrappedMasterService::GetVChunk> { + static constexpr const char* value = "GetVChunk"; +}; +template <> +struct RpcNameTraits<&WrappedMasterService::ReleaseVChunkReadLease> { + static constexpr const char* value = "ReleaseVChunkReadLease"; +}; +template <> +struct RpcNameTraits<&WrappedMasterService::RemoveVChunk> { + static constexpr const char* value = "RemoveVChunk"; +}; +template <> +struct RpcNameTraits<&WrappedMasterService::GetVChunkRuntimeInfo> { + static constexpr const char* value = "GetVChunkRuntimeInfo"; +}; + template <> struct RpcNameTraits<&WrappedMasterService::BatchPutRevoke> { static constexpr const char* value = "BatchPutRevoke"; @@ -249,6 +280,15 @@ template <> struct RpcNameTraits<&WrappedMasterService::PromotionObjectHeartbeat> { static constexpr const char* value = "PromotionObjectHeartbeat"; }; +template <> +struct RpcNameTraits<&WrappedMasterService::RemoveObjectHeartbeat> { + static constexpr const char* value = "RemoveObjectHeartbeat"; +}; + +template <> +struct RpcNameTraits<&WrappedMasterService::AckRemoveObjectHeartbeat> { + static constexpr const char* value = "AckRemoveObjectHeartbeat"; +}; template <> struct RpcNameTraits<&WrappedMasterService::PromotionAllocStart> { @@ -445,6 +485,45 @@ tl::expected MasterClient::invoke_rpc(Args&&... args) { return rpc_result; } +template +tl::expected MasterClient::invoke_rpc_to( + const std::string& address, Args&&... args) { + // 定向 RPC：用独立 targeted_accessor_ 按地址取 pool，不切换 + // client_accessor_ 的"当前地址"，避免与业务请求竞态。pool 取自返回值， + // 不依赖 targeted_accessor_ 的当前状态，因此可被多线程并发调用。 + auto pool = targeted_accessor_.GetOrCreateClientPool(address); + + if (metrics_) { + metrics_->rpc_count.inc({RpcNameTraits::value}); + } + + return async_simple::coro::syncAwait( + [&]() -> async_simple::coro::Lazy> { + auto ret = co_await pool->send_request( + [&](coro_io::client_reuse_hint, + coro_rpc::coro_rpc_client& client) { + return client.send_request( + std::forward(args)...); + }); + if (!ret.has_value()) { + co_return tl::make_unexpected(ErrorCode::RPC_FAIL); + } + auto result = co_await std::move(ret.value()); + if (!result) { + if (result.error().code == coro_rpc::errc::timed_out) { + co_return tl::make_unexpected(ErrorCode::RPC_TIMEOUT); + } + co_return tl::make_unexpected(ErrorCode::RPC_FAIL); + } + if constexpr (std::is_void_v) { + result->result(); + co_return tl::expected{}; + } else { + co_return std::move(result->result()); + } + }()); +} + template std::vector> MasterClient::invoke_batch_rpc( size_t input_size, Args&&... args) { @@ -569,11 +648,132 @@ ErrorCode MasterClient::Connect(const std::string& master_addr) { return ErrorCode::OK; } +ErrorCode MasterClient::LoadRoutingFromEtcd( + const std::string& etcd_endpoints, const std::string& cluster_namespace) { + if (etcd_endpoints.empty() || cluster_namespace.empty()) { + LOG(ERROR) << "LoadRoutingFromEtcd requires non-empty etcd_endpoints " + << "and cluster_namespace"; + return ErrorCode::INVALID_PARAMS; + } + + ErrorCode err = EtcdHelper::ConnectToEtcdStoreClient(etcd_endpoints); + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to connect etcd for partition routing: " << err; + return err; + } + + err = partition_router_.LoadFromEtcdSnapshot(cluster_namespace); + if (err != ErrorCode::OK) { + LOG(WARNING) << "Failed to load partition routing snapshot from etcd " + << "(namespace=" << cluster_namespace << "): " << err; + return err; + } + { + std::lock_guard lock(routing_config_mutex_); + routing_cluster_namespace_ = cluster_namespace; + } + LOG(INFO) << "Loaded partition routing from etcd: namespace=" + << cluster_namespace << " entries=" << partition_router_.Size(); + return ErrorCode::OK; +} + +ErrorCode MasterClient::RefreshSubmasterRouting() { + std::string cluster_namespace; + { + std::lock_guard lock(routing_config_mutex_); + cluster_namespace = routing_cluster_namespace_; + } + if (cluster_namespace.empty()) { + return ErrorCode::UNAVAILABLE_IN_CURRENT_STATUS; + } + return partition_router_.LoadFromEtcdSnapshot(cluster_namespace); +} + +std::optional MasterClient::ResolveSubmaster( + const std::string& key) const { + const uint16_t slot = partition::KvHashMap::Compute(tenant_id_, key); + auto submaster = partition_router_.ResolveSubmaster(slot); + if (submaster) { + LOG(INFO) << "ResolveSubmaster hit: tenant=" << tenant_id_.value() + << " key=" << key << " slot=" << slot + << " submaster=" << *submaster; + } else { + LOG(WARNING) << "ResolveSubmaster miss: tenant=" << tenant_id_.value() + << " key=" << key << " slot=" << slot + << " (routing not loaded or slot has no owner)"; + } + return submaster; +} + +void MasterClient::SwitchToSubmasterByAddress(const std::string& address) { + const std::string old_address = client_accessor_.GetAddress(); + client_accessor_.GetOrCreateClientPool(address); + if (old_address != address) { + LOG(INFO) << "SwitchToSubmasterByAddress: [" << old_address << "] -> [" + << address << "]"; + } +} + +ErrorCode MasterClient::SwitchToSubmaster(const std::string& tenant_id, + const std::string& key) { + // Routing not loaded (single-master mode): keep the current connection so + // existing behavior is preserved. Check Size() before ResolveSubmaster to + // avoid its per-miss WARNING log firing on every single-key request. + if (partition_router_.Size() == 0) { + return ErrorCode::OK; + } + + const TenantId tenant(tenant_id); + const uint16_t slot = partition::KvHashMap::Compute(tenant, key); + auto submaster = partition_router_.ResolveSubmaster(slot); + if (!submaster) { + LOG(WARNING) << "SwitchToSubmaster miss: tenant=" << tenant_id + << " key=" << key << " slot=" << slot + << " (slot has no owner in routing table)"; + return ErrorCode::UNAVAILABLE_IN_CURRENT_STATUS; + } + + SwitchToSubmasterByAddress(*submaster); + LOG(INFO) << "SwitchToSubmaster: tenant=" << tenant_id << " key=" << key + << " slot=" << slot << " -> submaster=" << *submaster; + return ErrorCode::OK; +} + +std::map> MasterClient::GroupKeysBySubmaster( + const std::vector& keys, const std::string& tenant_id) { + std::map> groups; + const TenantId tenant(tenant_id); + for (size_t i = 0; i < keys.size(); ++i) { + const uint16_t slot = partition::KvHashMap::Compute(tenant, keys[i]); + auto submaster = partition_router_.ResolveSubmaster(slot); + groups[submaster.value_or("")].push_back(i); + } + + std::string group_desc; + for (const auto& [submaster, indices] : groups) { + if (!group_desc.empty()) { + group_desc += ", "; + } + group_desc += (submaster.empty() ? std::string("") : submaster); + group_desc += ":" + std::to_string(indices.size()); + } + LOG(INFO) << "GroupKeysBySubmaster: tenant=" << tenant_id + << " total_keys=" << keys.size() << " groups=" << groups.size() + << " [" << group_desc << "]"; + return groups; +} + tl::expected MasterClient::ExistKey( const std::string& object_key) { ScopedVLogTimer timer(1, "MasterClient::ExistKey"); timer.LogRequest("object_key=", object_key); + ErrorCode switch_err = SwitchToSubmaster(tenant_id_.value(), object_key); + if (switch_err != ErrorCode::OK) { + timer.LogResponse("error_code=", switch_err); + return tl::make_unexpected(switch_err); + } + auto result = invoke_rpc<&WrappedMasterService::ExistKey, bool>( object_key, tenant_id_.value()); timer.LogResponseExpected(result); @@ -585,10 +785,43 @@ std::vector> MasterClient::BatchExistKey( ScopedVLogTimer timer(1, "MasterClient::BatchExistKey"); timer.LogRequest("keys_count=", object_keys.size()); - auto result = invoke_batch_rpc<&WrappedMasterService::BatchExistKey, bool>( - object_keys.size(), object_keys, tenant_id_.value()); - timer.LogResponse("result=", result.size(), " keys"); - return result; + // Single-master mode (routing not loaded): use the original batch path. + if (partition_router_.Size() == 0) { + auto result = + invoke_batch_rpc<&WrappedMasterService::BatchExistKey, bool>( + object_keys.size(), object_keys, tenant_id_.value()); + timer.LogResponse("result=", result.size(), " keys"); + return result; + } + + std::vector> results( + object_keys.size(), + tl::make_unexpected(ErrorCode::UNAVAILABLE_IN_CURRENT_STATUS)); + + auto groups = GroupKeysBySubmaster(object_keys, tenant_id_.value()); + for (auto& [submaster, indices] : groups) { + if (submaster.empty()) { + LOG(WARNING) << "BatchExistKey: " << indices.size() + << " key(s) have no submaster, marked unavailable"; + continue; + } + SwitchToSubmasterByAddress(submaster); + + std::vector group_keys; + group_keys.reserve(indices.size()); + for (size_t idx : indices) { + group_keys.push_back(object_keys[idx]); + } + + auto group_result = + invoke_batch_rpc<&WrappedMasterService::BatchExistKey, bool>( + group_keys.size(), group_keys, tenant_id_.value()); + for (size_t j = 0; j < indices.size(); ++j) { + results[indices[j]] = std::move(group_result[j]); + } + } + timer.LogResponse("result=", results.size(), " keys"); + return results; } tl::expected @@ -653,6 +886,12 @@ tl::expected MasterClient::GetReplicaList( ScopedVLogTimer timer(1, "MasterClient::GetReplicaList"); timer.LogRequest("object_key=", object_key, ", tenant_id=", tenant_id); + ErrorCode switch_err = SwitchToSubmaster(tenant_id, object_key); + if (switch_err != ErrorCode::OK) { + timer.LogResponse("error_code=", switch_err); + return tl::make_unexpected(switch_err); + } + const uint64_t trace_id = mooncake::logging::CurrentTraceId(); auto result = invoke_rpc<&WrappedMasterService::GetReplicaList, GetReplicaListResponse>(object_key, tenant_id, @@ -673,12 +912,48 @@ MasterClient::BatchGetReplicaList(const std::vector& object_keys, timer.LogRequest("keys_count=", object_keys.size(), ", tenant_id=", tenant_id); + // Single-master mode (routing not loaded): use the original batch path. + if (partition_router_.Size() == 0) { + const uint64_t trace_id = mooncake::logging::CurrentTraceId(); + auto result = + invoke_batch_rpc<&WrappedMasterService::BatchGetReplicaList, + GetReplicaListResponse>( + object_keys.size(), object_keys, tenant_id, trace_id, + client_id_); + timer.LogResponse("result=", result.size(), " operations"); + return result; + } + + std::vector> results( + object_keys.size(), + tl::make_unexpected(ErrorCode::UNAVAILABLE_IN_CURRENT_STATUS)); + const uint64_t trace_id = mooncake::logging::CurrentTraceId(); - auto result = invoke_batch_rpc<&WrappedMasterService::BatchGetReplicaList, - GetReplicaListResponse>( - object_keys.size(), object_keys, tenant_id, trace_id, client_id_); - timer.LogResponse("result=", result.size(), " operations"); - return result; + auto groups = GroupKeysBySubmaster(object_keys, tenant_id); + for (auto& [submaster, indices] : groups) { + if (submaster.empty()) { + LOG(WARNING) << "BatchGetReplicaList: " << indices.size() + << " key(s) have no submaster, marked unavailable"; + continue; + } + SwitchToSubmasterByAddress(submaster); + + std::vector group_keys; + group_keys.reserve(indices.size()); + for (size_t idx : indices) { + group_keys.push_back(object_keys[idx]); + } + + auto group_result = + invoke_batch_rpc<&WrappedMasterService::BatchGetReplicaList, + GetReplicaListResponse>( + group_keys.size(), group_keys, tenant_id, trace_id, client_id_); + for (size_t j = 0; j < indices.size(); ++j) { + results[indices[j]] = std::move(group_result[j]); + } + } + timer.LogResponse("result=", results.size(), " operations"); + return results; } tl::expected, ErrorCode> @@ -688,6 +963,12 @@ MasterClient::PutStart(const std::string& key, ScopedVLogTimer timer(1, "MasterClient::PutStart"); timer.LogRequest("key=", key, ", slice_count=", slice_lengths.size()); + ErrorCode switch_err = SwitchToSubmaster(tenant_id_.value(), key); + if (switch_err != ErrorCode::OK) { + timer.LogResponse("error_code=", switch_err); + return tl::make_unexpected(switch_err); + } + uint64_t total_slice_length = 0; for (const auto& slice_length : slice_lengths) { total_slice_length += slice_length; @@ -711,22 +992,60 @@ MasterClient::BatchPutStart( std::vector total_slice_lengths; total_slice_lengths.reserve(slice_lengths.size()); - for (const auto& slice_lengths : slice_lengths) { + for (const auto& per_key_lengths : slice_lengths) { uint64_t total_slice_length = 0; - for (const auto& slice_length : slice_lengths) { + for (const auto& slice_length : per_key_lengths) { total_slice_length += slice_length; } total_slice_lengths.emplace_back(total_slice_length); } const uint64_t trace_id = mooncake::logging::CurrentTraceId(); - auto result = invoke_batch_rpc<&WrappedMasterService::BatchPutStart, - std::vector>( - keys.size(), client_id_, keys, total_slice_lengths, config, - tenant_id_.value(), - trace_id); - timer.LogResponse("result=", result.size(), " operations"); - return result; + + // Single-master mode (routing not loaded): use the original batch path. + if (partition_router_.Size() == 0) { + auto result = + invoke_batch_rpc<&WrappedMasterService::BatchPutStart, + std::vector>( + keys.size(), client_id_, keys, total_slice_lengths, config, + tenant_id_.value(), trace_id); + timer.LogResponse("result=", result.size(), " operations"); + return result; + } + + std::vector, ErrorCode>> + results(keys.size(), + tl::make_unexpected(ErrorCode::UNAVAILABLE_IN_CURRENT_STATUS)); + + auto groups = GroupKeysBySubmaster(keys, tenant_id_.value()); + for (auto& [submaster, indices] : groups) { + if (submaster.empty()) { + LOG(WARNING) << "BatchPutStart: " << indices.size() + << " key(s) have no submaster, marked unavailable"; + continue; + } + SwitchToSubmasterByAddress(submaster); + + std::vector group_keys; + std::vector group_total_slice_lengths; + group_keys.reserve(indices.size()); + group_total_slice_lengths.reserve(indices.size()); + for (size_t idx : indices) { + group_keys.push_back(keys[idx]); + group_total_slice_lengths.push_back(total_slice_lengths[idx]); + } + + auto group_result = invoke_batch_rpc< + &WrappedMasterService::BatchPutStart, + std::vector>( + group_keys.size(), client_id_, group_keys, + group_total_slice_lengths, config, tenant_id_.value(), trace_id); + for (size_t j = 0; j < indices.size(); ++j) { + results[indices[j]] = std::move(group_result[j]); + } + } + timer.LogResponse("result=", results.size(), " operations"); + return results; } tl::expected MasterClient::PutEnd( @@ -734,6 +1053,13 @@ tl::expected MasterClient::PutEnd( ScopedVLogTimer timer(1, "MasterClient::PutEnd"); timer.LogRequest("key=", object_meta.key); + ErrorCode switch_err = + SwitchToSubmaster(tenant_id_.value(), object_meta.key); + if (switch_err != ErrorCode::OK) { + timer.LogResponse("error_code=", switch_err); + return tl::make_unexpected(switch_err); + } + const uint64_t trace_id = mooncake::logging::CurrentTraceId(); auto result = invoke_rpc<&WrappedMasterService::PutEnd, void>( client_id_, object_meta, replica_type, tenant_id_.value(), trace_id); @@ -747,11 +1073,52 @@ std::vector> MasterClient::BatchPutEnd( timer.LogRequest("keys_count=", object_metas.size()); const uint64_t trace_id = mooncake::logging::CurrentTraceId(); - auto result = invoke_batch_rpc<&WrappedMasterService::BatchPutEnd, void>( - object_metas.size(), client_id_, object_metas, replica_type, - tenant_id_.value(), trace_id); - timer.LogResponse("result=", result.size(), " operations"); - return result; + + // Single-master mode (routing not loaded): use the original batch path. + if (partition_router_.Size() == 0) { + auto result = + invoke_batch_rpc<&WrappedMasterService::BatchPutEnd, void>( + object_metas.size(), client_id_, object_metas, replica_type, + tenant_id_.value(), trace_id); + timer.LogResponse("result=", result.size(), " operations"); + return result; + } + + std::vector> results( + object_metas.size(), + tl::make_unexpected(ErrorCode::UNAVAILABLE_IN_CURRENT_STATUS)); + + std::vector keys; + keys.reserve(object_metas.size()); + for (const auto& meta : object_metas) { + keys.push_back(meta.key); + } + + auto groups = GroupKeysBySubmaster(keys, tenant_id_.value()); + for (auto& [submaster, indices] : groups) { + if (submaster.empty()) { + LOG(WARNING) << "BatchPutEnd: " << indices.size() + << " key(s) have no submaster, marked unavailable"; + continue; + } + SwitchToSubmasterByAddress(submaster); + + std::vector group_object_metas; + group_object_metas.reserve(indices.size()); + for (size_t idx : indices) { + group_object_metas.push_back(object_metas[idx]); + } + + auto group_result = + invoke_batch_rpc<&WrappedMasterService::BatchPutEnd, void>( + group_object_metas.size(), client_id_, group_object_metas, + replica_type, tenant_id_.value(), trace_id); + for (size_t j = 0; j < indices.size(); ++j) { + results[indices[j]] = std::move(group_result[j]); + } + } + timer.LogResponse("result=", results.size(), " operations"); + return results; } tl::expected MasterClient::PutRevoke( @@ -759,21 +1126,174 @@ tl::expected MasterClient::PutRevoke( ScopedVLogTimer timer(1, "MasterClient::PutRevoke"); timer.LogRequest("key=", key); + ErrorCode switch_err = SwitchToSubmaster(tenant_id_.value(), key); + if (switch_err != ErrorCode::OK) { + timer.LogResponse("error_code=", switch_err); + return tl::make_unexpected(switch_err); + } + auto result = invoke_rpc<&WrappedMasterService::PutRevoke, void>( client_id_, key, replica_type, tenant_id_.value()); timer.LogResponseExpected(result); return result; } +tl::expected MasterClient::VChunkPutStart( + const std::string& tenant_id, const std::string& key, uint64_t total_size, + int64_t now_ms) { + std::lock_guard routed_lock(vchunk_routed_rpc_mutex_); + const auto switch_err = SwitchToSubmaster(tenant_id, key); + if (switch_err != ErrorCode::OK) { + return tl::make_unexpected(switch_err); + } + auto result = invoke_rpc<&WrappedMasterService::VChunkPutStart, + VChunkMetadataRecord>(tenant_id, key, total_size, + now_ms); + if (!result && result.error() == ErrorCode::SLOT_NOT_OWNED && + RefreshSubmasterRouting() == ErrorCode::OK && + SwitchToSubmaster(tenant_id, key) == ErrorCode::OK) { + result = invoke_rpc<&WrappedMasterService::VChunkPutStart, + VChunkMetadataRecord>(tenant_id, key, total_size, + now_ms); + } + return result; +} + +tl::expected MasterClient::VChunkPutEnd( + const std::string& tenant_id, const std::string& key, + const std::string& vchunk_id, int64_t now_ms) { + std::lock_guard routed_lock(vchunk_routed_rpc_mutex_); + const auto switch_err = SwitchToSubmaster(tenant_id, key); + if (switch_err != ErrorCode::OK) { + return tl::make_unexpected(switch_err); + } + auto result = invoke_rpc<&WrappedMasterService::VChunkPutEnd, void>( + tenant_id, key, vchunk_id, now_ms); + if (!result && result.error() == ErrorCode::SLOT_NOT_OWNED && + RefreshSubmasterRouting() == ErrorCode::OK && + SwitchToSubmaster(tenant_id, key) == ErrorCode::OK) { + result = invoke_rpc<&WrappedMasterService::VChunkPutEnd, void>( + tenant_id, key, vchunk_id, now_ms); + } + return result; +} + +tl::expected MasterClient::VChunkPutRevoke( + const std::string& tenant_id, const std::string& key, + const std::string& vchunk_id) { + std::lock_guard routed_lock(vchunk_routed_rpc_mutex_); + const auto switch_err = SwitchToSubmaster(tenant_id, key); + if (switch_err != ErrorCode::OK) { + return tl::make_unexpected(switch_err); + } + auto result = invoke_rpc<&WrappedMasterService::VChunkPutRevoke, void>( + tenant_id, key, vchunk_id); + if (!result && result.error() == ErrorCode::SLOT_NOT_OWNED && + RefreshSubmasterRouting() == ErrorCode::OK && + SwitchToSubmaster(tenant_id, key) == ErrorCode::OK) { + result = invoke_rpc<&WrappedMasterService::VChunkPutRevoke, void>( + tenant_id, key, vchunk_id); + } + return result; +} + +tl::expected MasterClient::GetVChunk( + const std::string& tenant_id, const std::string& key) { + std::lock_guard routed_lock(vchunk_routed_rpc_mutex_); + const auto switch_err = SwitchToSubmaster(tenant_id, key); + if (switch_err != ErrorCode::OK) { + return tl::make_unexpected(switch_err); + } + auto result = invoke_rpc<&WrappedMasterService::GetVChunk, + VChunkReadLease>(tenant_id, key); + if (!result && result.error() == ErrorCode::SLOT_NOT_OWNED && + RefreshSubmasterRouting() == ErrorCode::OK && + SwitchToSubmaster(tenant_id, key) == ErrorCode::OK) { + result = invoke_rpc<&WrappedMasterService::GetVChunk, + VChunkReadLease>(tenant_id, key); + } + return result; +} + +tl::expected MasterClient::ReleaseVChunkReadLease( + const std::string& tenant_id, const std::string& key, + const std::string& lease_id) { + std::lock_guard routed_lock(vchunk_routed_rpc_mutex_); + const auto switch_err = SwitchToSubmaster(tenant_id, key); + if (switch_err != ErrorCode::OK) { + return tl::make_unexpected(switch_err); + } + return invoke_rpc<&WrappedMasterService::ReleaseVChunkReadLease, void>( + lease_id); +} + +tl::expected MasterClient::RemoveVChunk( + const std::string& tenant_id, const std::string& key, int64_t now_ms) { + std::lock_guard routed_lock(vchunk_routed_rpc_mutex_); + const auto switch_err = SwitchToSubmaster(tenant_id, key); + if (switch_err != ErrorCode::OK) { + return tl::make_unexpected(switch_err); + } + auto result = invoke_rpc<&WrappedMasterService::RemoveVChunk, void>( + tenant_id, key, now_ms); + if (!result && result.error() == ErrorCode::SLOT_NOT_OWNED && + RefreshSubmasterRouting() == ErrorCode::OK && + SwitchToSubmaster(tenant_id, key) == ErrorCode::OK) { + result = invoke_rpc<&WrappedMasterService::RemoveVChunk, void>( + tenant_id, key, now_ms); + } + return result; +} + +tl::expected MasterClient::GetVChunkRuntimeInfo() { + return invoke_rpc<&WrappedMasterService::GetVChunkRuntimeInfo, + VChunkRuntimeInfo>(); +} + std::vector> MasterClient::BatchPutRevoke( const std::vector& keys, ReplicaType replica_type) { ScopedVLogTimer timer(1, "MasterClient::BatchPutRevoke"); timer.LogRequest("keys_count=", keys.size()); - auto result = invoke_batch_rpc<&WrappedMasterService::BatchPutRevoke, void>( - keys.size(), client_id_, keys, replica_type, tenant_id_.value()); - timer.LogResponse("result=", result.size(), " operations"); - return result; + // Single-master mode (routing not loaded): use the original batch path. + if (partition_router_.Size() == 0) { + auto result = + invoke_batch_rpc<&WrappedMasterService::BatchPutRevoke, void>( + keys.size(), client_id_, keys, replica_type, + tenant_id_.value()); + timer.LogResponse("result=", result.size(), " operations"); + return result; + } + + std::vector> results( + keys.size(), + tl::make_unexpected(ErrorCode::UNAVAILABLE_IN_CURRENT_STATUS)); + + auto groups = GroupKeysBySubmaster(keys, tenant_id_.value()); + for (auto& [submaster, indices] : groups) { + if (submaster.empty()) { + LOG(WARNING) << "BatchPutRevoke: " << indices.size() + << " key(s) have no submaster, marked unavailable"; + continue; + } + SwitchToSubmasterByAddress(submaster); + + std::vector group_keys; + group_keys.reserve(indices.size()); + for (size_t idx : indices) { + group_keys.push_back(keys[idx]); + } + + auto group_result = + invoke_batch_rpc<&WrappedMasterService::BatchPutRevoke, void>( + group_keys.size(), client_id_, group_keys, replica_type, + tenant_id_.value()); + for (size_t j = 0; j < indices.size(); ++j) { + results[indices[j]] = std::move(group_result[j]); + } + } + timer.LogResponse("result=", results.size(), " operations"); + return results; } tl::expected, ErrorCode> @@ -783,6 +1303,12 @@ MasterClient::UpsertStart(const std::string& key, ScopedVLogTimer timer(1, "MasterClient::UpsertStart"); timer.LogRequest("key=", key, ", slice_count=", slice_lengths.size()); + ErrorCode switch_err = SwitchToSubmaster(tenant_id_.value(), key); + if (switch_err != ErrorCode::OK) { + timer.LogResponse("error_code=", switch_err); + return tl::make_unexpected(switch_err); + } + uint64_t total_slice_length = 0; for (const auto& slice_length : slice_lengths) { total_slice_length += slice_length; @@ -813,12 +1339,50 @@ MasterClient::BatchUpsertStart( total_slice_lengths.emplace_back(total); } - auto result = invoke_batch_rpc<&WrappedMasterService::BatchUpsertStart, - std::vector>( - keys.size(), client_id_, keys, total_slice_lengths, config, - tenant_id_.value()); - timer.LogResponse("result=", result.size(), " operations"); - return result; + // Single-master mode (routing not loaded): use the original batch path. + if (partition_router_.Size() == 0) { + auto result = + invoke_batch_rpc<&WrappedMasterService::BatchUpsertStart, + std::vector>( + keys.size(), client_id_, keys, total_slice_lengths, config, + tenant_id_.value()); + timer.LogResponse("result=", result.size(), " operations"); + return result; + } + + std::vector, ErrorCode>> + results(keys.size(), + tl::make_unexpected(ErrorCode::UNAVAILABLE_IN_CURRENT_STATUS)); + + auto groups = GroupKeysBySubmaster(keys, tenant_id_.value()); + for (auto& [submaster, indices] : groups) { + if (submaster.empty()) { + LOG(WARNING) << "BatchUpsertStart: " << indices.size() + << " key(s) have no submaster, marked unavailable"; + continue; + } + SwitchToSubmasterByAddress(submaster); + + std::vector group_keys; + std::vector group_total_slice_lengths; + group_keys.reserve(indices.size()); + group_total_slice_lengths.reserve(indices.size()); + for (size_t idx : indices) { + group_keys.push_back(keys[idx]); + group_total_slice_lengths.push_back(total_slice_lengths[idx]); + } + + auto group_result = invoke_batch_rpc< + &WrappedMasterService::BatchUpsertStart, + std::vector>( + group_keys.size(), client_id_, group_keys, + group_total_slice_lengths, config, tenant_id_.value()); + for (size_t j = 0; j < indices.size(); ++j) { + results[indices[j]] = std::move(group_result[j]); + } + } + timer.LogResponse("result=", results.size(), " operations"); + return results; } tl::expected MasterClient::UpsertEnd( @@ -826,6 +1390,13 @@ tl::expected MasterClient::UpsertEnd( ScopedVLogTimer timer(1, "MasterClient::UpsertEnd"); timer.LogRequest("key=", object_meta.key); + ErrorCode switch_err = + SwitchToSubmaster(tenant_id_.value(), object_meta.key); + if (switch_err != ErrorCode::OK) { + timer.LogResponse("error_code=", switch_err); + return tl::make_unexpected(switch_err); + } + auto result = invoke_rpc<&WrappedMasterService::UpsertEnd, void>( client_id_, object_meta, replica_type, tenant_id_.value()); timer.LogResponseExpected(result); @@ -837,10 +1408,51 @@ std::vector> MasterClient::BatchUpsertEnd( ScopedVLogTimer timer(1, "MasterClient::BatchUpsertEnd"); timer.LogRequest("keys_count=", object_metas.size()); - auto result = invoke_batch_rpc<&WrappedMasterService::BatchUpsertEnd, void>( - object_metas.size(), client_id_, object_metas, tenant_id_.value()); - timer.LogResponse("result=", result.size(), " operations"); - return result; + // Single-master mode (routing not loaded): use the original batch path. + if (partition_router_.Size() == 0) { + auto result = + invoke_batch_rpc<&WrappedMasterService::BatchUpsertEnd, void>( + object_metas.size(), client_id_, object_metas, + tenant_id_.value()); + timer.LogResponse("result=", result.size(), " operations"); + return result; + } + + std::vector> results( + object_metas.size(), + tl::make_unexpected(ErrorCode::UNAVAILABLE_IN_CURRENT_STATUS)); + + std::vector keys; + keys.reserve(object_metas.size()); + for (const auto& meta : object_metas) { + keys.push_back(meta.key); + } + + auto groups = GroupKeysBySubmaster(keys, tenant_id_.value()); + for (auto& [submaster, indices] : groups) { + if (submaster.empty()) { + LOG(WARNING) << "BatchUpsertEnd: " << indices.size() + << " key(s) have no submaster, marked unavailable"; + continue; + } + SwitchToSubmasterByAddress(submaster); + + std::vector group_object_metas; + group_object_metas.reserve(indices.size()); + for (size_t idx : indices) { + group_object_metas.push_back(object_metas[idx]); + } + + auto group_result = + invoke_batch_rpc<&WrappedMasterService::BatchUpsertEnd, void>( + group_object_metas.size(), client_id_, group_object_metas, + tenant_id_.value()); + for (size_t j = 0; j < indices.size(); ++j) { + results[indices[j]] = std::move(group_result[j]); + } + } + timer.LogResponse("result=", results.size(), " operations"); + return results; } tl::expected MasterClient::UpsertRevoke( @@ -848,6 +1460,12 @@ tl::expected MasterClient::UpsertRevoke( ScopedVLogTimer timer(1, "MasterClient::UpsertRevoke"); timer.LogRequest("key=", key); + ErrorCode switch_err = SwitchToSubmaster(tenant_id_.value(), key); + if (switch_err != ErrorCode::OK) { + timer.LogResponse("error_code=", switch_err); + return tl::make_unexpected(switch_err); + } + auto result = invoke_rpc<&WrappedMasterService::UpsertRevoke, void>( client_id_, key, replica_type, tenant_id_.value()); timer.LogResponseExpected(result); @@ -859,11 +1477,43 @@ std::vector> MasterClient::BatchUpsertRevoke( ScopedVLogTimer timer(1, "MasterClient::BatchUpsertRevoke"); timer.LogRequest("keys_count=", keys.size()); - auto result = - invoke_batch_rpc<&WrappedMasterService::BatchUpsertRevoke, void>( - keys.size(), client_id_, keys, tenant_id_.value()); - timer.LogResponse("result=", result.size(), " operations"); - return result; + // Single-master mode (routing not loaded): use the original batch path. + if (partition_router_.Size() == 0) { + auto result = + invoke_batch_rpc<&WrappedMasterService::BatchUpsertRevoke, void>( + keys.size(), client_id_, keys, tenant_id_.value()); + timer.LogResponse("result=", result.size(), " operations"); + return result; + } + + std::vector> results( + keys.size(), + tl::make_unexpected(ErrorCode::UNAVAILABLE_IN_CURRENT_STATUS)); + + auto groups = GroupKeysBySubmaster(keys, tenant_id_.value()); + for (auto& [submaster, indices] : groups) { + if (submaster.empty()) { + LOG(WARNING) << "BatchUpsertRevoke: " << indices.size() + << " key(s) have no submaster, marked unavailable"; + continue; + } + SwitchToSubmasterByAddress(submaster); + + std::vector group_keys; + group_keys.reserve(indices.size()); + for (size_t idx : indices) { + group_keys.push_back(keys[idx]); + } + + auto group_result = + invoke_batch_rpc<&WrappedMasterService::BatchUpsertRevoke, void>( + group_keys.size(), client_id_, group_keys, tenant_id_.value()); + for (size_t j = 0; j < indices.size(); ++j) { + results[indices[j]] = std::move(group_result[j]); + } + } + timer.LogResponse("result=", results.size(), " operations"); + return results; } tl::expected MasterClient::Remove(const std::string& key, @@ -871,6 +1521,12 @@ tl::expected MasterClient::Remove(const std::string& key, ScopedVLogTimer timer(1, "MasterClient::Remove"); timer.LogRequest("key=", key, ", force=", force); + ErrorCode switch_err = SwitchToSubmaster(tenant_id_.value(), key); + if (switch_err != ErrorCode::OK) { + timer.LogResponse("error_code=", switch_err); + return tl::make_unexpected(switch_err); + } + auto result = invoke_rpc<&WrappedMasterService::Remove, void>( key, force, tenant_id_.value()); timer.LogResponseExpected(result); @@ -903,10 +1559,42 @@ std::vector> MasterClient::BatchRemove( ScopedVLogTimer timer(1, "MasterClient::BatchRemove"); timer.LogRequest("keys_count=", keys.size(), ", force=", force); - auto result = invoke_batch_rpc<&WrappedMasterService::BatchRemove, void>( - keys.size(), keys, force, tenant_id_.value()); - timer.LogResponse("result=", result.size(), " operations"); - return result; + // Single-master mode (routing not loaded): use the original batch path. + if (partition_router_.Size() == 0) { + auto result = invoke_batch_rpc<&WrappedMasterService::BatchRemove, void>( + keys.size(), keys, force, tenant_id_.value()); + timer.LogResponse("result=", result.size(), " operations"); + return result; + } + + std::vector> results( + keys.size(), + tl::make_unexpected(ErrorCode::UNAVAILABLE_IN_CURRENT_STATUS)); + + auto groups = GroupKeysBySubmaster(keys, tenant_id_.value()); + for (auto& [submaster, indices] : groups) { + if (submaster.empty()) { + LOG(WARNING) << "BatchRemove: " << indices.size() + << " key(s) have no submaster, marked unavailable"; + continue; + } + SwitchToSubmasterByAddress(submaster); + + std::vector group_keys; + group_keys.reserve(indices.size()); + for (size_t idx : indices) { + group_keys.push_back(keys[idx]); + } + + auto group_result = + invoke_batch_rpc<&WrappedMasterService::BatchRemove, void>( + group_keys.size(), group_keys, force, tenant_id_.value()); + for (size_t j = 0; j < indices.size(); ++j) { + results[indices[j]] = std::move(group_result[j]); + } + } + timer.LogResponse("result=", results.size(), " operations"); + return results; } tl::expected MasterClient::MountSegment( @@ -922,6 +1610,18 @@ tl::expected MasterClient::MountSegment( return result; } +tl::expected MasterClient::MountSegmentTo( + const std::string& address, const Segment& segment) { + ScopedVLogTimer timer(1, "MasterClient::MountSegmentTo"); + timer.LogRequest("address=", address, ", id=", segment.id, + ", client_id=", client_id_); + + auto result = invoke_rpc_to<&WrappedMasterService::MountSegment, void>( + address, segment, client_id_); + timer.LogResponseExpected(result); + return result; +} + tl::expected MasterClient::MountNoFSegment( const NoFSegment& segment) { ScopedVLogTimer timer(1, "MasterClient::MountNofSegment"); @@ -970,6 +1670,18 @@ tl::expected MasterClient::UnmountSegment( return result; } +tl::expected MasterClient::UnmountSegmentTo( + const std::string& address, const UUID& segment_id) { + ScopedVLogTimer timer(1, "MasterClient::UnmountSegmentTo"); + timer.LogRequest("address=", address, ", segment_id=", segment_id, + ", client_id=", client_id_); + + auto result = invoke_rpc_to<&WrappedMasterService::UnmountSegment, void>( + address, segment_id, client_id_); + timer.LogResponseExpected(result); + return result; +} + tl::expected MasterClient::GracefulUnmountSegment( const UUID& segment_id, uint64_t grace_period_ms) { ScopedVLogTimer timer(1, "MasterClient::GracefulUnmountSegment"); @@ -983,6 +1695,21 @@ tl::expected MasterClient::GracefulUnmountSegment( return result; } +tl::expected MasterClient::GracefulUnmountSegmentTo( + const std::string& address, const UUID& segment_id, + uint64_t grace_period_ms) { + ScopedVLogTimer timer(1, "MasterClient::GracefulUnmountSegmentTo"); + timer.LogRequest("address=", address, ", segment_id=", segment_id, + ", client_id=", client_id_, + ", grace_period_ms=", grace_period_ms); + + auto result = + invoke_rpc_to<&WrappedMasterService::GracefulUnmountSegment, void>( + address, segment_id, client_id_, grace_period_ms); + timer.LogResponseExpected(result); + return result; +} + tl::expected MasterClient::UnmountNoFSegment( const UUID& segment_id) { ScopedVLogTimer timer(1, "MasterClient::UnmountNoFSegment"); @@ -1038,6 +1765,21 @@ tl::expected MasterClient::Ping() { return result; } +std::string MasterClient::GetCurrentAddress() const { + return client_accessor_.GetAddress(); +} + +tl::expected MasterClient::PingTo( + const std::string& address) { + ScopedVLogTimer timer(1, "MasterClient::PingTo"); + timer.LogRequest("address=", address, ", client_id=", client_id_); + + auto result = invoke_rpc_to<&WrappedMasterService::Ping, PingResponse>( + address, client_id_); + timer.LogResponseExpected(result); + return result; +} + tl::expected MasterClient::GetFsdir() { ScopedVLogTimer timer(1, "MasterClient::GetFsdir"); timer.LogRequest("action=get_fsdir"); @@ -1058,6 +1800,18 @@ tl::expected MasterClient::QuerySegmentStatusById( return result; } +tl::expected +MasterClient::QuerySegmentStatusByIdTo(const std::string& address, + const UUID& segment_id) { + ScopedVLogTimer timer(1, "MasterClient::QuerySegmentStatusByIdTo"); + timer.LogRequest("address=", address, ", segment_id=", segment_id); + + auto result = invoke_rpc_to<&WrappedMasterService::QuerySegmentStatusById, + SegmentStatus>(address, segment_id); + timer.LogResponseExpected(result); + return result; +} + tl::expected MasterClient::GetStorageConfig() { ScopedVLogTimer timer(1, "MasterClient::GetStorageConfig"); @@ -1196,6 +1950,23 @@ MasterClient::PromotionObjectHeartbeat(const UUID& client_id) { std::vector>(client_id); } +tl::expected, ErrorCode> +MasterClient::RemoveObjectHeartbeat(const UUID& client_id) { + ScopedVLogTimer timer(1, "MasterClient::RemoveObjectHeartbeat"); + timer.LogRequest("client_id=", client_id.first, ":", client_id.second); + return invoke_rpc<&WrappedMasterService::RemoveObjectHeartbeat, + std::vector>(client_id); +} + +tl::expected MasterClient::AckRemoveObjectHeartbeat( + const UUID& client_id, const std::vector& tasks) { + ScopedVLogTimer timer(1, "MasterClient::AckRemoveObjectHeartbeat"); + timer.LogRequest("client_id=", client_id.first, ":", client_id.second, + " tasks=", tasks.size()); + return invoke_rpc<&WrappedMasterService::AckRemoveObjectHeartbeat, void>( + client_id, tasks); +} + tl::expected MasterClient::PromotionAllocStart( const UUID& client_id, const std::string& key, uint64_t size, @@ -1421,11 +2192,45 @@ std::vector> MasterClient::BatchEvictDiskReplica( timer.LogRequest("keys_count=", keys.size(), ", tenant_id=", tenant_id, ", replica_type=", replica_type); - auto result = - invoke_batch_rpc<&WrappedMasterService::BatchEvictDiskReplica, void>( - keys.size(), client_id_, keys, tenant_id, replica_type); - timer.LogResponse("result=", result.size(), " operations"); - return result; + // Single-master mode (routing not loaded): use the original batch path. + if (partition_router_.Size() == 0) { + auto result = + invoke_batch_rpc<&WrappedMasterService::BatchEvictDiskReplica, + void>(keys.size(), client_id_, keys, tenant_id, + replica_type); + timer.LogResponse("result=", result.size(), " operations"); + return result; + } + + std::vector> results( + keys.size(), + tl::make_unexpected(ErrorCode::UNAVAILABLE_IN_CURRENT_STATUS)); + + auto groups = GroupKeysBySubmaster(keys, tenant_id); + for (auto& [submaster, indices] : groups) { + if (submaster.empty()) { + LOG(WARNING) << "BatchEvictDiskReplica: " << indices.size() + << " key(s) have no submaster, marked unavailable"; + continue; + } + SwitchToSubmasterByAddress(submaster); + + std::vector group_keys; + group_keys.reserve(indices.size()); + for (size_t idx : indices) { + group_keys.push_back(keys[idx]); + } + + auto group_result = + invoke_batch_rpc<&WrappedMasterService::BatchEvictDiskReplica, + void>(group_keys.size(), client_id_, group_keys, + tenant_id, replica_type); + for (size_t j = 0; j < indices.size(); ++j) { + results[indices[j]] = std::move(group_result[j]); + } + } + timer.LogResponse("result=", results.size(), " operations"); + return results; } } // namespace mooncake diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index d5660e8e74..03d890f099 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -1,6 +1,5 @@ #include "master_service.h" -#include #include #include #include @@ -11,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -38,6 +38,8 @@ #ifdef STORE_USE_ETCD #include "etcd_helper.h" #include "ha/kv/etcd_ha_kv_backend.h" +#include "cvm/etcd_view_store.h" +#include "cvm/cvm_keys.h" #endif #include "ha/oplog/oplog_batch_storage.h" #include "ha/oplog/ordered_oplog_writer.h" @@ -198,6 +200,10 @@ MasterService::MasterService(const MasterServiceConfig& config) config.ha_backend_type == "etcd"), oplog_batch_max_entries_(config.oplog_batch_max_entries), cluster_id_(config.cluster_id), + master_id_(config.master_id), + cvm_http_port_(config.cvm_http_port), + cvm_http_host_(config.cvm_http_host), + submaster_count_(config.submaster_count), root_fs_dir_(config.root_fs_dir), global_file_segment_size_(config.global_file_segment_size), enable_disk_eviction_(config.enable_disk_eviction), @@ -207,6 +213,10 @@ MasterService::MasterService(const MasterServiceConfig& config) tenant_quota_connector_uri_(config.tenant_quota_connector_uri), segment_manager_(config.memory_allocator, config.enable_cxl), nof_segment_manager_(config.memory_allocator), + vchunk_manager_(config.vchunk_config, config.vchunk_metadata_store), + vchunk_enabled_(config.vchunk_config.enabled), + vchunk_reaper_interval_ms_(config.vchunk_config.reaper_interval_ms), + vchunk_reaper_max_scan_(config.vchunk_config.reaper_max_scan), memory_allocator_type_(config.memory_allocator), allocation_strategy_type_(config.enable_cxl ? AllocationStrategyType::CXL @@ -229,6 +239,20 @@ MasterService::MasterService(const MasterServiceConfig& config) offloading_queue_limit_(config.offloading_queue_limit), offload_cap_ratio_(config.offload_cap_ratio), task_manager_(config.task_manager_config) { + const bool partitioned_vchunk = + config.vchunk_config.enabled && config.enable_ha && + config.ha_backend_type == "etcd" && config.submaster_count > 1; + if (config.vchunk_config.enabled && config.vchunk_metadata_store) { + if (partitioned_vchunk) { + vchunk_recovery_pending_ = true; + } else { + const auto error = + vchunk_manager_.Recover(getCurrentTimeInMilli()); + if (error != ErrorCode::OK) { + throw std::runtime_error("failed to recover vchunk metadata"); + } + } + } // Initialize HTTP metadata key prefix (read env var once at startup) const char* custom_prefix = std::getenv("MC_METADATA_CLUSTER_ID"); if (custom_prefix && std::strlen(custom_prefix) > 0) { @@ -449,6 +473,13 @@ MasterService::MasterService(const MasterServiceConfig& config) #endif } + // KV partition (CVM) ownership is now owned by the HA supervisor: the + // supervisor creates the CvmController (etcd lease + master registration + + // snapshot aggregation + membership loop), then injects the lease id and + // drives SlotOwnerHeartbeat around serve start/stop via + // SetCvmLeaseId() / StartSlotOwnerHeartbeat() / StopSlotOwnerHeartbeat(). + // MasterService itself only publishes slot/segment ownership records. + eviction_running_ = true; eviction_thread_ = std::thread(&MasterService::EvictionThreadFunc, this); VLOG(1) << "action=start_eviction_thread"; @@ -492,79 +523,1045 @@ MasterService::MasterService(const MasterServiceConfig& config) } } - if (enable_snapshot_ && !enable_oplog_) { - if (memory_allocator_type_ == BufferAllocatorType::OFFSET) { - // Initialize and start snapshot manager - MasterSnapshotManagerOptions snapshot_options; - snapshot_options.enable_snapshot = enable_snapshot_; - snapshot_options.snapshot_interval_seconds = - snapshot_interval_seconds_; - snapshot_options.snapshot_child_timeout_seconds = - snapshot_child_timeout_seconds_; - snapshot_options.snapshot_retention_count = - snapshot_retention_count_; - snapshot_options.snapshot_backup_dir = snapshot_backup_dir_; - snapshot_options.use_snapshot_backup_dir = use_snapshot_backup_dir_; - snapshot_options.snapshot_catalog_store_type = - snapshot_catalog_store_type_; - snapshot_options.snapshot_catalog_store_connstring = - snapshot_catalog_store_connstring_; - snapshot_options.ha_backend_type = ha_backend_type_; - snapshot_options.ha_backend_connstring = ha_backend_connstring_; - snapshot_options.cluster_id = cluster_id_; - snapshot_options.enable_ha = enable_ha_; + if (enable_snapshot_ && !enable_oplog_) { + if (memory_allocator_type_ == BufferAllocatorType::OFFSET) { + // Initialize and start snapshot manager + MasterSnapshotManagerOptions snapshot_options; + snapshot_options.enable_snapshot = enable_snapshot_; + snapshot_options.snapshot_interval_seconds = + snapshot_interval_seconds_; + snapshot_options.snapshot_child_timeout_seconds = + snapshot_child_timeout_seconds_; + snapshot_options.snapshot_retention_count = + snapshot_retention_count_; + snapshot_options.snapshot_backup_dir = snapshot_backup_dir_; + snapshot_options.use_snapshot_backup_dir = use_snapshot_backup_dir_; + snapshot_options.snapshot_catalog_store_type = + snapshot_catalog_store_type_; + snapshot_options.snapshot_catalog_store_connstring = + snapshot_catalog_store_connstring_; + snapshot_options.ha_backend_type = ha_backend_type_; + snapshot_options.ha_backend_connstring = ha_backend_connstring_; + snapshot_options.cluster_id = cluster_id_; + snapshot_options.enable_ha = enable_ha_; + + snapshot_manager_ = std::make_unique( + this, snapshot_options, snapshot_mutex_, + snapshot_object_store_.get(), snapshot_catalog_store_.get()); + snapshot_manager_->Start(); + } + } else if (enable_snapshot_ && enable_oplog_) { + LOG(INFO) << "Skipping primary snapshot generation in batch-record " + "OpLog mode; snapshots are owned by standby"; + } + + if (enable_cxl_) { + allocation_strategy_ = std::make_shared(); + segment_manager_.initializeCxlAllocator(cxl_path_, cxl_size_); + VLOG(1) << "action=start_cxl_global_allocator"; + } + if (vchunk_enabled_ && !vchunk_recovery_pending_) { + StartVChunkReaper(); + } +} + +tl::expected MasterService::VChunkPutStart( + const TenantId& tenant_id, const std::string& key, uint64_t total_size, + bool is_ssd_segment, int64_t now_ms, + const std::set& excluded_segments) { + if (!vchunk_enabled_) { + return tl::make_unexpected(ErrorCode::UNAVAILABLE_IN_CURRENT_MODE); + } + if (!OwnsVChunkSlot(cvm::KeySlot(tenant_id, key))) { + return tl::make_unexpected(ErrorCode::SLOT_NOT_OWNED); + } + auto allocator_access = segment_manager_.getAllocatorAccess(); + return vchunk_manager_.PutStart(allocator_access.getAllocatorManager(), + tenant_id, key, total_size, + is_ssd_segment, now_ms, + excluded_segments); +} + +ErrorCode MasterService::VChunkPutEnd(const TenantId& tenant_id, + const std::string& key, + const std::string& vchunk_id, + int64_t now_ms) { + if (!vchunk_enabled_) { + return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + } + if (!OwnsVChunkSlot(cvm::KeySlot(tenant_id, key))) { + return ErrorCode::SLOT_NOT_OWNED; + } + return vchunk_manager_.PutEnd(tenant_id, key, vchunk_id, now_ms); +} + +ErrorCode MasterService::VChunkPutRevoke(const TenantId& tenant_id, + const std::string& key, + const std::string& vchunk_id) { + if (!vchunk_enabled_) { + return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + } + if (!OwnsVChunkSlot(cvm::KeySlot(tenant_id, key))) { + return ErrorCode::SLOT_NOT_OWNED; + } + auto allocator_access = segment_manager_.getAllocatorAccess(); + return vchunk_manager_.PutRevoke(tenant_id, key, vchunk_id); +} + +tl::expected MasterService::GetVChunk( + const TenantId& tenant_id, const std::string& key) const { + if (!vchunk_enabled_) { + return tl::make_unexpected(ErrorCode::UNAVAILABLE_IN_CURRENT_MODE); + } + if (!OwnsVChunkSlot(cvm::KeySlot(tenant_id, key))) { + return tl::make_unexpected(ErrorCode::SLOT_NOT_OWNED); + } + return vchunk_manager_.Get(tenant_id, key); +} + +tl::expected +MasterService::AcquireVChunkRead(const TenantId& tenant_id, + const std::string& key) const { + if (!vchunk_enabled_) { + return tl::make_unexpected(ErrorCode::UNAVAILABLE_IN_CURRENT_MODE); + } + if (!OwnsVChunkSlot(cvm::KeySlot(tenant_id, key))) { + return tl::make_unexpected(ErrorCode::SLOT_NOT_OWNED); + } + return vchunk_manager_.AcquireRead(tenant_id, key); +} + +tl::expected +MasterService::AcquireVChunkReadLease(const TenantId& tenant_id, + const std::string& key, + int64_t now_ms) { + constexpr int64_t kRemoteReadLeaseTtlMs = 5 * 60 * 1000; + if (now_ms < 0 || + now_ms > std::numeric_limits::max() - + kRemoteReadLeaseTtlMs) { + return tl::make_unexpected(ErrorCode::INVALID_PARAMS); + } + auto handle = AcquireVChunkRead(tenant_id, key); + if (!handle) { + return tl::make_unexpected(handle.error()); + } + VChunkReadLease lease{handle->record(), UuidToString(generate_uuid())}; + { + std::lock_guard guard(vchunk_read_leases_mutex_); + vchunk_read_leases_.emplace( + lease.lease_id, + VChunkRemoteReadLease{std::move(*handle), + now_ms + kRemoteReadLeaseTtlMs}); + } + return lease; +} + +ErrorCode MasterService::ReleaseVChunkReadLease( + const std::string& lease_id) { + if (lease_id.empty()) { + return ErrorCode::INVALID_PARAMS; + } + std::lock_guard guard(vchunk_read_leases_mutex_); + vchunk_read_leases_.erase(lease_id); + return ErrorCode::OK; +} + +ErrorCode MasterService::RemoveVChunk(const TenantId& tenant_id, + const std::string& key, + int64_t now_ms) { + if (!vchunk_enabled_) { + return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + } + if (!OwnsVChunkSlot(cvm::KeySlot(tenant_id, key))) { + return ErrorCode::SLOT_NOT_OWNED; + } + auto allocator_access = segment_manager_.getAllocatorAccess(); + return vchunk_manager_.Remove(tenant_id, key, now_ms); +} + +VChunkRuntimeInfo MasterService::GetVChunkRuntimeInfo() const { + return {vchunk_enabled_, vchunk_manager_.HasPersistentMetadata()}; +} + +tl::expected MasterService::ReapExpiredVChunks( + int64_t now_ms, size_t max_scan) { + if (!vchunk_enabled_) { + return tl::make_unexpected(ErrorCode::UNAVAILABLE_IN_CURRENT_MODE); + } + { + std::lock_guard guard(vchunk_read_leases_mutex_); + std::erase_if(vchunk_read_leases_, [now_ms](const auto& item) { + return item.second.expires_at_ms <= now_ms; + }); + } + auto allocator_access = segment_manager_.getAllocatorAccess(); + return vchunk_manager_.ReapExpired( + now_ms, max_scan, [this](const VChunkMetadataRecord& record) { + return OwnsVChunkSlot( + cvm::KeySlot(TenantId(record.tenant_id), record.key)); + }); +} + +VChunkMetricsSnapshot MasterService::GetVChunkMetrics() const { + return vchunk_manager_.MetricsSnapshot(); +} + +void MasterService::VChunkReaperThreadFunc() { + std::unique_lock lock(vchunk_reaper_mutex_); + while (vchunk_reaper_running_) { + if (vchunk_reaper_cv_.wait_for( + lock, std::chrono::milliseconds(vchunk_reaper_interval_ms_), + [this] { return !vchunk_reaper_running_.load(); })) { + break; + } + lock.unlock(); + const auto result = ReapExpiredVChunks(getCurrentTimeInMilli(), + vchunk_reaper_max_scan_); + if (!result) { + LOG(ERROR) << "vchunk reaper failed, error=" + << static_cast(result.error()); + } + lock.lock(); + } +} + +void MasterService::StartVChunkReaper() { + bool expected = false; + if (!vchunk_reaper_running_.compare_exchange_strong(expected, true)) { + return; + } + vchunk_reaper_thread_ = + std::thread(&MasterService::VChunkReaperThreadFunc, this); +} + +std::unique_ptr +MasterService::CreateSnapshotCatalogStore() { + auto catalog_kind = ParseSnapshotCatalogKind(snapshot_catalog_store_type_); + if (!catalog_kind) { + throw std::invalid_argument(catalog_kind.error()); + } + + switch (catalog_kind.value()) { + case SnapshotCatalogBackendKind::kEmbedded: + return std::make_unique< + ha::backends::embedded::EmbeddedSnapshotCatalogStore>( + snapshot_object_store_.get(), cluster_id_); + case SnapshotCatalogBackendKind::kRedis: { +#ifndef STORE_USE_REDIS + throw std::invalid_argument( + "redis snapshot catalog store is unavailable in the current " + "build"); +#else + const auto connstring = !snapshot_catalog_store_connstring_.empty() + ? snapshot_catalog_store_connstring_ + : ha_backend_connstring_; + if (connstring.empty()) { + throw std::invalid_argument( + "redis snapshot catalog store requires a connection " + "string"); + } + return std::make_unique< + ha::backends::redis::RedisSnapshotCatalogStore>( + snapshot_object_store_.get(), connstring, cluster_id_); +#endif + } + } + + throw std::invalid_argument("unknown snapshot catalog store type"); +} + +void MasterService::SetCvmLeaseId(EtcdLeaseId lease_id) { + cvm_lease_id_ = lease_id; +} + +void MasterService::StopSlotOwnerHeartbeat() { + if (slot_owner_heartbeat_) { + slot_owner_heartbeat_->Stop(); + slot_owner_heartbeat_.reset(); + } +} + +#ifdef STORE_USE_ETCD +ErrorCode MasterService::StartSlotOwnerHeartbeat() { + const bool kv_partition_enabled = enable_ha_ && + ha_backend_type_ == "etcd" && + !master_id_.empty() && + !cluster_id_.empty(); + if (!kv_partition_enabled) { + return ErrorCode::OK; + } + + // The supervisor's CvmController already connected the etcd client before + // this point; ConnectToEtcdStoreClient is idempotent so re-connecting is a + // safe no-op for direct constructions / tests. + ErrorCode connect_err = + EtcdHelper::ConnectToEtcdStoreClient(ha_backend_connstring_); + if (connect_err != ErrorCode::OK) { + LOG(WARNING) << "StartSlotOwnerHeartbeat: failed to connect etcd: " + << connect_err; + return connect_err; + } + + if (slot_owner_heartbeat_) { + return ErrorCode::OK; // already running + } + + const auto initial_slots = ResolveOwnedSlotsForCvm(); + UpdateOwnedSlots(initial_slots); + if (vchunk_recovery_pending_) { + const auto error = vchunk_manager_.Recover( + getCurrentTimeInMilli(), [this](const VChunkMetadataRecord& record) { + return OwnsVChunkSlot( + cvm::KeySlot(TenantId(record.tenant_id), record.key)); + }); + if (error != ErrorCode::OK) { + LOG(ERROR) << "Failed to recover owned vchunk metadata: " + << static_cast(error); + return error; + } + vchunk_recovery_pending_ = false; + StartVChunkReaper(); + } + + cvm::SlotOwnerHeartbeat::Config hb_config; + hb_config.cluster_namespace = cluster_id_; + hb_config.master_id = master_id_; + // Dynamic partition: recompute the owned slot set from the etcd master + // membership on every heartbeat so multiple submaster instances split the + // 16384 slots without overwriting each other. + hb_config.dynamic_slot_resolver = [this]() { + auto slots = ResolveOwnedSlotsForCvm(); + UpdateOwnedSlots(slots); + return slots; + }; + hb_config.lease_id = cvm_lease_id_; + // live primary → live primary 元数据交接（P4 技术债 1）：slot 平移时在 + // 释放端导出对象元数据、在获得端导入，避免只依赖 standby 回放晋升路径。 + hb_config.on_slot_acquired = [this](uint16_t slot) { + (void)ImportSlotMetadata(slot); + }; + hb_config.on_slot_released = [this](uint16_t slot) { + (void)ExportSlotMetadata(slot); + }; + const bool lease_bound = hb_config.lease_id != 0; + slot_owner_heartbeat_ = + std::make_unique(std::move(hb_config)); + ErrorCode hb_err = slot_owner_heartbeat_->Start(); + if (hb_err != ErrorCode::OK) { + LOG(WARNING) << "Failed to start SlotOwnerHeartbeat: " << hb_err; + slot_owner_heartbeat_.reset(); + return hb_err; + } + LOG(INFO) << "Started SlotOwnerHeartbeat: master_id=" << master_id_ + << ", cluster_namespace=" << cluster_id_ + << ", dynamic_partition=true" + << ", lease_bound=" << lease_bound; + return ErrorCode::OK; +} + +#ifdef STORE_USE_ETCD +ErrorCode MasterService::StartInterMasterRpc() { + const bool cvm_enabled = enable_ha_ && ha_backend_type_ == "etcd" && + !master_id_.empty() && !cluster_id_.empty(); + if (!cvm_enabled) { + return ErrorCode::OK; + } + + // The supervisor's CvmController already connected the etcd client; + // re-connecting is an idempotent no-op. + ErrorCode connect_err = + EtcdHelper::ConnectToEtcdStoreClient(ha_backend_connstring_); + if (connect_err != ErrorCode::OK) { + LOG(WARNING) << "StartInterMasterRpc: failed to connect etcd: " + << connect_err; + return connect_err; + } + + if (inter_master_rpc_) { + return ErrorCode::OK; // already running + } + + inter_master_rpc_ = std::make_unique(); + ErrorCode rc = inter_master_rpc_->Start(cluster_id_, master_id_); + if (rc != ErrorCode::OK) { + LOG(WARNING) << "StartInterMasterRpc: refresh loop not started: " + << rc << " (manual member updates still work)"; + // Keep the client object for manual member updates; only the + // etcd-driven refresh thread is unavailable. + } + LOG(INFO) << "Started InterMasterRpcClient: master_id=" << master_id_ + << ", cluster_namespace=" << cluster_id_; + return ErrorCode::OK; +} + +void MasterService::StopInterMasterRpc() { + if (inter_master_rpc_) { + inter_master_rpc_->Stop(); + inter_master_rpc_.reset(); + } +} +#else +ErrorCode MasterService::StartInterMasterRpc() { return ErrorCode::OK; } +void MasterService::StopInterMasterRpc() {} +#endif + +uint32_t MasterService::GetOwnedSlotCount() const { + std::shared_lock lock(owned_slots_mutex_); + if (!owned_slots_ready_) { + return 0; + } + return static_cast( + std::count(owned_slot_lookup_.begin(), owned_slot_lookup_.end(), true)); +} + +tl::expected, ErrorCode> +MasterService::InterMasterAllocateReplicas( + const std::string& tenant_id, const std::string& key, + uint64_t slice_length, uint64_t replica_num, + const std::vector& preferred_segments) { + if (key.empty() || slice_length == 0 || replica_num == 0) { + return tl::make_unexpected(ErrorCode::INVALID_PARAMS); + } + + ScopedAllocatorAccess allocator_access = + segment_manager_.getAllocatorAccess(); + const auto& allocator_manager = allocator_access.getAllocatorManager(); + const auto& local_names = allocator_manager.getNames(); + if (local_names.empty()) { + return tl::make_unexpected(ErrorCode::NO_AVAILABLE_HANDLE); + } + + // Strict mode: when preferred segments are given, only allocate in them. + // Exclude every other local segment so the strategy's random fallback + // cannot leak the allocation into unrelated segments. + std::set excluded; + if (!preferred_segments.empty()) { + const std::unordered_set preferred( + preferred_segments.begin(), preferred_segments.end()); + bool any_preferred_local = false; + for (const auto& name : local_names) { + if (preferred.count(name) > 0) { + any_preferred_local = true; + } else { + excluded.insert(name); + } + } + if (!any_preferred_local) { + return tl::make_unexpected(ErrorCode::NO_AVAILABLE_HANDLE); + } + } + + auto allocation = allocation_strategy_->Allocate( + allocator_manager, slice_length, replica_num, preferred_segments, + excluded, ReplicaType::MEMORY); + if (!allocation.has_value()) { + return tl::make_unexpected(allocation.error()); + } + // Partial allocation is not forwarded: the slot owner requires the full + // replica count. Dropped here => destructors free the partial handles. + if (allocation->size() != replica_num) { + return tl::make_unexpected(ErrorCode::NO_AVAILABLE_HANDLE); + } + + std::vector descriptors; + descriptors.reserve(allocation->size()); + for (const auto& replica : allocation.value()) { + descriptors.push_back(replica.get_descriptor()); + } + const std::string scoped_key = TenantId(tenant_id).MakeScopedKey(key); + { + std::lock_guard lock(inter_master_keepalive_mutex_); + // Replace any stale keepalive entry for the same key: erasing frees + // the old handles (idempotent re-PutStart after an aborted write). + inter_master_keepalive_.erase(scoped_key); + inter_master_keepalive_.emplace(scoped_key, + std::move(allocation.value())); + } + LOG(INFO) << "InterMasterAllocateReplicas: allocated " + << descriptors.size() << " replica(s) for scoped_key=" + << scoped_key << ", slice_length=" << slice_length + << ", preferred_segments=" << preferred_segments.size(); + return descriptors; +} + +tl::expected MasterService::InterMasterFreeReplicas( + const std::string& tenant_id, const std::string& key) { + const std::string scoped_key = TenantId(tenant_id).MakeScopedKey(key); + std::vector released; + { + std::lock_guard lock(inter_master_keepalive_mutex_); + auto it = inter_master_keepalive_.find(scoped_key); + if (it == inter_master_keepalive_.end()) { + return false; // not the owner of this key's handles + } + released = std::move(it->second); + inter_master_keepalive_.erase(it); + } + // Replica destructors free the handles at this (segment owning) master. + LOG(INFO) << "InterMasterFreeReplicas: freed " << released.size() + << " replica(s) for scoped_key=" << scoped_key; + return true; +} + +tl::expected +MasterService::InterMasterGetReplicaList(const std::string& key, + const std::string& tenant_id) { + const TenantId tenant(tenant_id); + const auto object_id = MakeObjectIdentityForRequest(key, tenant); + // peer 互信：不校验 OwnsSlot、不再次转发，直接本地查询（转发链止于 + // 第一跳，避免视图不一致时的循环转发）。 + return GetReplicaListLocal(object_id); +} + +std::vector> +MasterService::InterMasterBatchGetReplicaList( + const std::vector& keys, const std::string& tenant_id) { + const TenantId tenant(tenant_id); + return BatchGetReplicaListLocal(keys, tenant); +} + +tl::expected, ErrorCode> +MasterService::InterMasterPutStart( + const UUID& client_id, const std::string& key, const std::string& tenant_id, + uint64_t slice_length, const ReplicateConfig& config) { + const TenantId tenant(tenant_id); + // peer 互信：调用方已按 slot 归属解析本机为 owner，直接执行完整本地 + // PutStart（分配 + 写元数据 + keepalive）。本机 OwnsSlot==true，不会再 + // 二次转发（转发链止于第一跳，避免视图不一致时的循环转发）。 + return PutStart(client_id, key, tenant, slice_length, config); +} + +tl::expected, ErrorCode> +MasterService::InterMasterUpsertStart( + const UUID& client_id, const std::string& key, const std::string& tenant_id, + uint64_t slice_length, const ReplicateConfig& config) { + const TenantId tenant(tenant_id); + // Upsert 转发：本机为 slot owner，执行完整本地 UpsertStart 以保留 + // "已存在则覆盖（preemption）"语义；由 PutStart 转发走 PutStart 会丢失 + // 该覆盖语义。peer 互信，不二次转发。 + return UpsertStart(client_id, key, tenant, slice_length, config); +} + +tl::expected, ErrorCode> +MasterService::TryAllocateReplicasRemotely( + const std::string& key, const TenantId& tenant_id, uint64_t value_length, + size_t replica_num, const std::vector& preferred_segments) { +#ifdef STORE_USE_ETCD + if (!inter_master_rpc_ || replica_num == 0) { + return tl::make_unexpected(ErrorCode::NO_AVAILABLE_HANDLE); + } + const std::string tenant_str = tenant_id.value(); + for (const auto& member : inter_master_rpc_->GetMembers()) { + if (member.master_id.empty() || member.master_id == master_id_) { + continue; + } + auto remote = inter_master_rpc_->AllocateReplicas( + member.master_id, tenant_str, key, value_length, + static_cast(replica_num), preferred_segments); + if (!remote.has_value()) { + VLOG(1) << "Remote allocation on " << member.master_id + << " failed for key=" << key << ": " + << toString(remote.error()); + continue; + } + if (remote->size() < replica_num) { + // Partial result: free it at the peer and keep probing. + inter_master_rpc_->FreeReplicas(member.master_id, tenant_str, key); + continue; + } + + // Materialize dummy-allocator replicas from the descriptors (same + // pattern as ImportSlotMetadata). The real handles stay alive at the + // segment owner's keepalive registry. + std::vector replicas; + replicas.reserve(remote->size()); + for (const auto& desc : remote.value()) { + if (!desc.is_memory_replica()) { + continue; + } + const auto& mem_desc = desc.get_memory_descriptor(); + const std::string& endpoint = + mem_desc.buffer_descriptor.transport_endpoint_; + std::shared_ptr alloc; + { + std::lock_guard lock( + remote_replica_allocator_keepalive_mutex_); + auto& slot = remote_replica_allocator_keepalive_[endpoint]; + if (!slot) { + slot = std::make_shared(endpoint, + endpoint); + } + alloc = slot; + } + replicas.emplace_back( + std::make_unique( + alloc, mem_desc.buffer_descriptor), + desc.status); + } + if (replicas.size() != replica_num) { + // Unexpected descriptor types: undo at the peer and fail. + inter_master_rpc_->FreeReplicas(member.master_id, tenant_str, key); + continue; + } + + { + std::lock_guard lock(remote_allocated_keys_mutex_); + remote_allocated_keys_.insert(tenant_id.MakeScopedKey(key)); + } + LOG(INFO) << "Allocated " << replicas.size() + << " remote replica(s) for key=" << key + << " on segment owner " << member.master_id; + return replicas; + } + return tl::make_unexpected(ErrorCode::NO_AVAILABLE_HANDLE); +#else + (void)key; + (void)tenant_id; + (void)value_length; + (void)replica_num; + (void)preferred_segments; + return tl::make_unexpected(ErrorCode::NO_AVAILABLE_HANDLE); +#endif +} + +void MasterService::EnqueueRemoteFreeIfTracked(const TenantId& tenant_id, + const std::string& key, + QuotaEraseMode quota_mode) { +#ifdef STORE_USE_ETCD + { + std::lock_guard lock(remote_allocated_keys_mutex_); + if (remote_allocated_keys_.erase(tenant_id.MakeScopedKey(key)) == 0) { + return; // handles (if any) are local + } + } + if (quota_mode == QuotaEraseMode::kHandoff) { + // Slot migration: the data bytes live on. The importing master + // re-registers the key and owns the remote free from now on. + return; + } + if (inter_master_rpc_) { + inter_master_rpc_->EnqueueBroadcastFree(tenant_id.value(), key); + } +#endif +} + +ErrorCode MasterService::ExportSlotMetadata(uint16_t slot) { + // 1. Collect this slot's object metadata across all shards. + SlotMetadataExport export_payload; + export_payload.slot = slot; + export_payload.source_master_id = master_id_; + + for (size_t shard_idx = 0; shard_idx < kNumShards; ++shard_idx) { + MetadataShardAccessorRO shard(this, shard_idx); + for (const auto& [tenant_id, tenant_state] : shard->tenants) { + for (const auto& [user_key, metadata] : tenant_state.metadata) { + if (cvm::KeySlot(tenant_id, user_key) != slot) { + continue; + } + StandbyObjectEntry entry; + entry.tenant_id = tenant_id.value(); + entry.key = user_key; + entry.metadata.client_id = metadata.client_id; + entry.metadata.size = metadata.size; + entry.metadata.group_id = metadata.group_id; + entry.metadata.data_type = metadata.data_type; + const auto& replicas = metadata.GetAllReplicas(); + entry.metadata.replicas.reserve(replicas.size()); + for (const auto& replica : replicas) { + entry.metadata.replicas.push_back(replica.get_descriptor()); + } + export_payload.objects.push_back(std::move(entry)); + } + } + } + + // 2. Serialize (struct_pack) and persist as a binary etcd value. + auto bytes = struct_pack::serialize(export_payload); + std::string value(bytes.begin(), bytes.end()); + const std::string key = cvm::SlotMetadataExportKey(cluster_id_, slot); + ErrorCode err = + EtcdHelper::Put(key.data(), key.size(), value.data(), value.size()); + if (err != ErrorCode::OK) { + LOG(WARNING) << "ExportSlotMetadata: put failed slot=" << slot + << ", objects=" << export_payload.objects.size() + << ", err=" << err; + return err; + } + + // 3. Drop the exported objects from local metadata (release cleanup). + for (size_t shard_idx = 0; shard_idx < kNumShards; ++shard_idx) { + MetadataShardAccessorRW shard(this, shard_idx); + for (auto tenant_it = shard->tenants.begin(); + tenant_it != shard->tenants.end();) { + auto& tenant_state = tenant_it->second; + for (auto it = tenant_state.metadata.begin(); + it != tenant_state.metadata.end();) { + if (cvm::KeySlot(tenant_it->first, it->first) == slot) { + it = EraseMetadata(tenant_state, it, tenant_it->first, + QuotaEraseMode::kHandoff); + } else { + ++it; + } + } + if (tenant_state.Empty()) { + tenant_it = shard->tenants.erase(tenant_it); + } else { + ++tenant_it; + } + } + } + + LOG(INFO) << "Exported slot metadata: slot=" << slot + << ", objects=" << export_payload.objects.size() + << ", bytes=" << value.size() << ", source=" << master_id_; + return ErrorCode::OK; +} + +ErrorCode MasterService::ImportSlotMetadata(uint16_t slot) { + const std::string key = cvm::SlotMetadataExportKey(cluster_id_, slot); + std::string value; + EtcdRevisionId revision = 0; + ErrorCode err = EtcdHelper::Get(key.data(), key.size(), value, revision); + if (err == ErrorCode::ETCD_KEY_NOT_EXIST) { + // No graceful export available (e.g. the previous owner died and the + // slot was reclaimed via standby replay). Nothing to pull. + return ErrorCode::OK; + } + if (err != ErrorCode::OK) { + LOG(WARNING) << "ImportSlotMetadata: get failed slot=" << slot + << ", err=" << err; + return err; + } + + SlotMetadataExport export_payload; + if (struct_pack::deserialize_to(export_payload, value) != + struct_pack::errc::ok) { + LOG(ERROR) << "ImportSlotMetadata: deserialize failed slot=" << slot + << ", bytes=" << value.size(); + return ErrorCode::DESERIALIZE_FAIL; + } + + const auto resolve = [](const StandbyObjectEntry& entry) { + auto [scoped_tenant_id, user_key] = TenantId::ParseScopedKey(entry.key); + TenantId tenant_id(entry.tenant_id); + if (tenant_id.IsDefault() && !scoped_tenant_id.IsDefault()) { + tenant_id = std::move(scoped_tenant_id); + } + return std::make_pair(std::move(tenant_id), std::move(user_key)); + }; + + std::unordered_map> + objects_by_shard; + for (const auto& entry : export_payload.objects) { + auto [tenant_id, user_key] = resolve(entry); + if (!tenant_id.IsValid()) { + LOG(WARNING) << "ImportSlotMetadata: invalid tenant for slot=" + << slot << ", key=" << entry.key; + continue; + } + const auto shard_idx = entry.metadata.group_id.empty() + ? getShardIndex(tenant_id, user_key) + : getShardIndex(entry.metadata.group_id); + objects_by_shard[shard_idx].push_back(&entry); + } + + size_t imported = 0; + for (const auto& [shard_idx, shard_objects] : objects_by_shard) { + MetadataShardAccessorRW shard(this, shard_idx); + auto now = std::chrono::system_clock::now(); + for (const auto* entry_ptr : shard_objects) { + const auto& entry = *entry_ptr; + auto [tenant_id, user_key] = resolve(entry); + const auto& standby_meta = entry.metadata; + std::vector replicas; + replicas.reserve(standby_meta.replicas.size()); + for (const auto& desc : standby_meta.replicas) { + if (desc.is_memory_replica()) { + const auto& mem_desc = desc.get_memory_descriptor(); + const std::string& endpoint = + mem_desc.buffer_descriptor.transport_endpoint_; + auto& alloc = standby_allocator_keepalive_[endpoint]; + if (!alloc) { + alloc = std::make_shared( + endpoint, endpoint); + } + replicas.emplace_back( + std::make_unique( + alloc, mem_desc.buffer_descriptor), + desc.status); + } else if (desc.is_nof_replica()) { + const auto& nof_desc = desc.get_nof_descriptor(); + const std::string& endpoint = + nof_desc.buffer_descriptor.transport_endpoint_; + auto& alloc = standby_allocator_keepalive_[endpoint]; + if (!alloc) { + alloc = std::make_shared( + endpoint, endpoint); + } + replicas.emplace_back( + std::make_unique( + alloc, nof_desc.buffer_descriptor), + desc.status, ReplicaType::NOF_SSD); + } else if (desc.is_disk_replica()) { + const auto& disk_desc = desc.get_disk_descriptor(); + replicas.emplace_back(disk_desc.file_path, + disk_desc.object_size, desc.status); + } else if (desc.is_local_disk_replica()) { + const auto& local_disk_desc = + desc.get_local_disk_descriptor(); + replicas.emplace_back(local_disk_desc.client_id, + local_disk_desc.object_size, + local_disk_desc.transport_endpoint, + desc.status); + } + } + + auto& tenant_state = shard->tenants[tenant_id]; + auto [metadata_it, inserted] = tenant_state.metadata.emplace( + std::piecewise_construct, std::forward_as_tuple(user_key), + std::forward_as_tuple( + standby_meta.client_id, now, standby_meta.size, + std::move(replicas), false, false, standby_meta.data_type, + standby_meta.group_id, tenant_id, user_key)); + if (!inserted) { + // 新获得 slot 时理论上不应碰撞；若碰撞则跳过以避免重复记账。 + LOG(WARNING) << "ImportSlotMetadata: duplicate key slot=" << slot + << ", key=" << entry.key << ", skipped"; + continue; + } + auto& metadata = metadata_it->second; + if (!standby_meta.group_id.empty()) { + RegisterGroupMember(tenant_state, tenant_id, user_key, + standby_meta.group_id); + } + tenant_state.processing_keys.erase(user_key); + + // A2：补回交接对象的账务（object count、cache 计数、quota），与 + // Export 侧 kHandoff 的释放对称，避免新 primary 账务缺失。 + IncrementTenantMetadataObjectCount(tenant_id); + SyncCacheTotalAccounting(metadata); + const uint64_t committed_charge = + CompletedMemoryQuotaCharge(metadata); + metadata.reserved_quota_charge_bytes = 0; + metadata.pending_replaced_quota_charge_bytes = 0; + metadata.committed_quota_charge_bytes = 0; + if (committed_charge > 0) { + auto reserve_result = + ReserveTenantQuota(tenant_id, committed_charge); + if (reserve_result) { + CommitTenantQuota(tenant_id, committed_charge); + metadata.committed_quota_charge_bytes = committed_charge; + } else { + LOG(WARNING) + << "ImportSlotMetadata: quota reserve failed tenant=" + << tenant_id.value() << ", bytes=" << committed_charge + << ", err=" << reserve_result.error(); + } + } + ++imported; + } + } + + LOG(INFO) << "Imported slot metadata: slot=" << slot + << ", objects=" << imported + << ", source=" << export_payload.source_master_id; + + // 技术债 3.3：一次性交接完成后删除 slot_meta 键，避免二进制导出残留。 + // 删除失败仅告警不阻断——残留键会在下次 acquire 时被幂等重导。 + const std::string end = cvm::PrefixEnd(key); + ErrorCode del_err = + EtcdHelper::DeleteRange(key.data(), key.size(), end.data(), end.size()); + if (del_err != ErrorCode::OK) { + LOG(WARNING) << "ImportSlotMetadata: delete slot_meta failed slot=" + << slot << ", err=" << del_err; + } + return ErrorCode::OK; +} +#else +ErrorCode MasterService::StartSlotOwnerHeartbeat() { return ErrorCode::OK; } + +ErrorCode MasterService::ExportSlotMetadata(uint16_t /*slot*/) { + return ErrorCode::OK; +} + +ErrorCode MasterService::ImportSlotMetadata(uint16_t /*slot*/) { + return ErrorCode::OK; +} +#endif + +#ifdef STORE_USE_ETCD +void MasterService::PublishSegmentOwnerForCvm(const Segment& segment) { + if (!enable_ha_ || ha_backend_type_ != "etcd" || master_id_.empty() || + cluster_id_.empty()) { + return; + } + cvm::SegmentOwner owner; + owner.segment_id = UuidToString(segment.id); + owner.owner_master_id = master_id_; + owner.state = static_cast(cvm::SegmentOwnerState::kStable); + + // Bind to the supervisor-owned CvmController's lease so segment ownership + // is auto-removed when this master dies; fall back to a persistent record + // when no lease has been injected yet. + ErrorCode err; + if (cvm_lease_id_ != 0) { + err = cvm::EtcdViewStore::SaveSegmentOwnerWithLease( + cluster_id_, owner, cvm_lease_id_); + } else { + err = cvm::EtcdViewStore::SaveSegmentOwner(cluster_id_, owner); + } + if (err != ErrorCode::OK) { + LOG(WARNING) << "PublishSegmentOwnerForCvm SaveSegmentOwner failed: " + "segment=" + << owner.segment_id << " err=" << err; + } +} + +void MasterService::RemoveSegmentOwnerForCvm(const UUID& segment_id) { + if (!enable_ha_ || ha_backend_type_ != "etcd" || master_id_.empty() || + cluster_id_.empty()) { + return; + } + const std::string id = UuidToString(segment_id); + ErrorCode err = cvm::EtcdViewStore::DeleteSegmentOwner(cluster_id_, id); + if (err != ErrorCode::OK) { + LOG(WARNING) << "RemoveSegmentOwnerForCvm DeleteSegmentOwner failed: " + "segment=" + << id << " err=" << err; + } +} - snapshot_manager_ = std::make_unique( - this, snapshot_options, snapshot_mutex_, - snapshot_object_store_.get(), snapshot_catalog_store_.get()); - snapshot_manager_->Start(); +std::vector MasterService::ResolveOwnedSlotsForCvm() { + // etcd 读取失败：sticky 策略——沿用上一轮成功解析的结果（可能为空）。 + // 绝不回退为全量接管：瞬时抖动引发的全量认领会与对端产生覆盖战 + // （双方反复重写对方 slot 记录，导致 slot 分布持续震荡不收敛）。 + std::vector masters; + ViewVersionId version; + ErrorCode err = + cvm::EtcdViewStore::LoadAllMasters(cluster_id_, masters, version); + if (err != ErrorCode::OK) { + std::lock_guard lock(cvm_resolver_mutex_); + LOG(WARNING) << "ResolveOwnedSlotsForCvm: LoadAllMasters failed: " << err + << ", keeping previous owned set (sticky), count=" + << cvm_last_resolved_owned_slots_.size(); + return cvm_last_resolved_owned_slots_; + } + + std::vector ids; + ids.reserve(masters.size()); + for (const auto& m : masters) { + // Only serving primaries own slots. Standbys do not publish slot + // ownership, so including them would strand part of the slot space. + if (static_cast(m.role) != cvm::MasterRole::kPrimary) { + continue; + } + if (!m.master_id.empty()) { + ids.push_back(m.master_id); } - } else if (enable_snapshot_ && enable_oplog_) { - LOG(INFO) << "Skipping primary snapshot generation in batch-record " - "OpLog mode; snapshots are owned by standby"; } + std::sort(ids.begin(), ids.end()); + ids.erase(std::unique(ids.begin(), ids.end()), ids.end()); - if (enable_cxl_) { - allocation_strategy_ = std::make_shared(); - segment_manager_.initializeCxlAllocator(cxl_path_, cxl_size_); - VLOG(1) << "action=start_cxl_global_allocator"; + // etcd 注册表是唯一事实源：本机必须已注册为 primary 才参与分配。 + // 不再无条件注入本机 master_id——否则任一节点视图缺失对端时都会以 + // n==1 身份全量接管，是 slot 覆盖战的直接根源。 + const bool self_registered = + std::binary_search(ids.begin(), ids.end(), master_id_); + std::vector slots; + if (self_registered) { + // 技术债 3.1：一致性哈希环分配已提取到 cvm::ResolveOwnedSlotsOnRing + // 纯函数（可单元测试），本机 owned slots 由 (ids, master_id_) 决定。 + slots = cvm::ResolveOwnedSlotsOnRing(ids, master_id_); + } else { + LOG(WARNING) << "ResolveOwnedSlotsForCvm: self not registered as " + "primary in etcd (masters=" + << ids.size() << "), owning no slots this cycle"; + } + + { + std::lock_guard lock(cvm_resolver_mutex_); + cvm_last_resolved_owned_slots_ = slots; } + return slots; } -std::unique_ptr -MasterService::CreateSnapshotCatalogStore() { - auto catalog_kind = ParseSnapshotCatalogKind(snapshot_catalog_store_type_); - if (!catalog_kind) { - throw std::invalid_argument(catalog_kind.error()); +std::optional MasterService::ResolveSlotOwnerMasterId( + uint16_t slot) const { + std::vector masters; + ViewVersionId version; + ErrorCode err = + cvm::EtcdViewStore::LoadAllMasters(cluster_id_, masters, version); + if (err != ErrorCode::OK) { + LOG(WARNING) << "ResolveSlotOwnerMasterId: LoadAllMasters failed: " + << err << ", slot=" << slot; + return std::nullopt; } - switch (catalog_kind.value()) { - case SnapshotCatalogBackendKind::kEmbedded: - return std::make_unique< - ha::backends::embedded::EmbeddedSnapshotCatalogStore>( - snapshot_object_store_.get(), cluster_id_); - case SnapshotCatalogBackendKind::kRedis: { -#ifndef STORE_USE_REDIS - throw std::invalid_argument( - "redis snapshot catalog store is unavailable in the current " - "build"); + std::vector ids; + ids.reserve(masters.size()); + for (const auto& m : masters) { + if (static_cast(m.role) != cvm::MasterRole::kPrimary) { + continue; + } + if (!m.master_id.empty()) { + ids.push_back(m.master_id); + } + } + std::sort(ids.begin(), ids.end()); + ids.erase(std::unique(ids.begin(), ids.end()), ids.end()); + + // 与 ResolveOwnedSlotsForCvm 保持一致：纯粹以 etcd 注册表计算环， + // 不注入本机 master_id，保证所有权判定与认领发布使用同一个环。 + // 解析结果为空（注册表无 primary）时由调用方跳过转发。 + return cvm::ResolveSlotOwnerOnRing(ids, slot); +} #else - const auto connstring = !snapshot_catalog_store_connstring_.empty() - ? snapshot_catalog_store_connstring_ - : ha_backend_connstring_; - if (connstring.empty()) { - throw std::invalid_argument( - "redis snapshot catalog store requires a connection " - "string"); - } - return std::make_unique< - ha::backends::redis::RedisSnapshotCatalogStore>( - snapshot_object_store_.get(), connstring, cluster_id_); +void MasterService::PublishSegmentOwnerForCvm(const Segment&) {} +void MasterService::RemoveSegmentOwnerForCvm(const UUID&) {} +std::vector MasterService::ResolveOwnedSlotsForCvm() { return {}; } +std::optional MasterService::ResolveSlotOwnerMasterId( + uint16_t /*slot*/) const { + return std::nullopt; +} #endif + +void MasterService::UpdateOwnedSlots(const std::vector& slots) { + std::unique_lock lock(owned_slots_mutex_); + owned_slot_lookup_.assign(cvm::kSlotCount, false); + for (uint16_t slot : slots) { + if (slot < cvm::kSlotCount) { + owned_slot_lookup_[slot] = true; } } + owned_slots_ready_ = true; + // 一致性哈希环分配结果（3.1）：仅在数量变化（含首次解析）时打印， + // 记录本机当前负责的 slot 规模；心跳周期内重复不变则静默。 + if (!owned_slot_count_logged_ || slots.size() != last_logged_owned_count_) { + LOG(INFO) << "Owned slots updated (consistent-hash ring): count=" + << slots.size() << "/" << cvm::kSlotCount + << (owned_slot_count_logged_ ? " (changed)" : ""); + owned_slot_count_logged_ = true; + last_logged_owned_count_ = slots.size(); + } + VLOG(1) << "UpdateOwnedSlots: " << slots.size() << " slots"; +} - throw std::invalid_argument("unknown snapshot catalog store type"); +bool MasterService::OwnsSlot(uint16_t slot) const { + std::shared_lock lock(owned_slots_mutex_); + if (!owned_slots_ready_) { + return true; // partition 未启用或尚未解析过，放行 + } + return slot < owned_slot_lookup_.size() && owned_slot_lookup_[slot]; +} + +bool MasterService::OwnsVChunkSlot(uint16_t slot) const { + std::shared_lock lock(owned_slots_mutex_); + if (!owned_slots_ready_) { + const bool partitioned = enable_ha_ && ha_backend_type_ == "etcd" && + submaster_count_ > 1; + return !partitioned; + } + return slot < owned_slot_lookup_.size() && owned_slot_lookup_[slot]; } MasterService::~MasterService() { @@ -572,6 +1569,10 @@ MasterService::~MasterService() { ordered_oplog_writer_->Stop(); } + // Stop the SlotOwnerHeartbeat thread before tearing down the rest. + StopSlotOwnerHeartbeat(); + StopInterMasterRpc(); + // Stop and join the threads eviction_running_ = false; client_monitor_running_ = false; @@ -584,6 +1585,7 @@ MasterService::~MasterService() { task_cleanup_running_ = false; job_dispatch_running_ = false; http_metadata_cleanup_running_ = false; + vchunk_reaper_running_ = false; graceful_unmount_scheduler_.Stop(); #ifdef USE_NOF nof_heartbeat_running_ = false; @@ -592,6 +1594,7 @@ MasterService::~MasterService() { // Wake sleepers so join() doesn't block for long sleep intervals. task_cleanup_cv_.notify_all(); http_metadata_cleanup_cv_.notify_all(); + vchunk_reaper_cv_.notify_all(); if (eviction_thread_.joinable()) { eviction_thread_.join(); @@ -613,6 +1616,9 @@ MasterService::~MasterService() { if (job_dispatch_thread_.joinable()) { job_dispatch_thread_.join(); } + if (vchunk_reaper_thread_.joinable()) { + vchunk_reaper_thread_.join(); + } // Reset snapshot manager after all other threads have joined // This triggers the destructor which joins the snapshot thread @@ -834,7 +1840,27 @@ auto MasterService::MountSegment(const Segment& segment, const UUID& client_id) } } - if (enable_oplog_ && ordered_oplog_writer_) { + // 幂等重挂载（segment 已存在）同样要把 client 标记为 OK：Ping 仅凭 + // ok_client_ 判定 client_status，若此处不标记，HeartbeatAllSubmasters + // 收到 NEED_REMOUNT 后用普通 mount 重挂永远无法消除该状态，形成 + // 每秒一次的 remount 死循环（并持续写 OpLog/segment_view）。 + // 注意：必须在 segment 访问释放之后再加 client_mutex_，避免与 + // ClientMonitorFunc 的 client_mutex_ -> segment access 加锁顺序倒挂。 + if (mount_result == ErrorCode::SEGMENT_ALREADY_EXISTS) { + std::unique_lock client_lock(client_mutex_); + if (ok_client_.find(client_id) == ok_client_.end()) { + ok_client_.insert(client_id); + MasterMetricManager::instance().inc_active_clients(); + LOG(INFO) << "client_id=" << client_id + << ", action=mount_segment_idempotent_ok" + << ", segment_name=" << segment.name; + } + } + + // 幂等重挂载不产生新的元数据变更，segment 首次挂载时已记录 + // SEGMENT_MOUNT OpLog，重复追加只会让 OpLog 无谓增长。 + if (enable_oplog_ && ordered_oplog_writer_ && + mount_result != ErrorCode::SEGMENT_ALREADY_EXISTS) { SegmentMountOp op; op.segment_name = segment.name; op.transport_endpoint = segment.te_endpoint; @@ -850,6 +1876,7 @@ auto MasterService::MountSegment(const Segment& segment, const UUID& client_id) if (mount_result == ErrorCode::OK) { RecomputeTenantEffectiveQuotas(); } + PublishSegmentOwnerForCvm(segment); return {}; } @@ -1227,6 +2254,10 @@ auto MasterService::ReMountSegment(const std::vector& segments, } RecomputeTenantEffectiveQuotas(); + for (const auto& seg : segments) { + PublishSegmentOwnerForCvm(seg); + } + return {}; } @@ -1764,6 +2795,14 @@ void MasterService::FinalizeRemovedReplicasAfterDurable( const bool erased_local_disk = std::any_of( erased_replicas.begin(), erased_replicas.end(), [](const Replica& replica) { return replica.is_local_disk_replica(); }); + std::vector local_disk_holders; + for (const auto& replica : erased_replicas) { + if (!replica.is_local_disk_replica()) continue; + auto client_id = replica.get_local_disk_client_id(); + if (client_id.has_value()) { + local_disk_holders.push_back(client_id.value()); + } + } ReleaseLocalDiskUsage(erased_replicas); if (erased_local_disk) { shard.OnDiskReplicaRemoved(erased_local_disk, metadata); @@ -1774,6 +2813,11 @@ void MasterService::FinalizeRemovedReplicasAfterDurable( shard->tenants.erase(tenant_it); } } + if (erased_local_disk) { + EnqueueRemoveTasks( + local_disk_holders, + RemoveTaskItem{tenant_id.value(), durable_entry.object_key}); + } } void MasterService::FinalizeMetadataEraseAfterDurable( @@ -1810,6 +2854,7 @@ void MasterService::FinalizeExpiredProcessingReplicasAfterDurable( } auto& metadata = accessor.Get(); + auto replicas = PopReplicasWithCacheTotalAccounting( metadata, &Replica::fn_is_processing); if (!replicas.empty()) { @@ -2041,10 +3086,15 @@ MasterService::EraseMetadata( tenant_state.replication_tasks.erase(key); ErasePromotionTaskIfPresent(tenant_state, key, tenant_id); - ReleaseLocalDiskUsage(metadata.GetAllReplicas()); + // kHandoff 跳过 ReleaseLocalDiskUsage：数据字节仍留在共享 segment， + // 不应在此扣减 ssd_used_bytes（否则造成账务偏差）。 + if (quota_mode != QuotaEraseMode::kHandoff) { + ReleaseLocalDiskUsage(metadata.GetAllReplicas()); + } AccountCacheTotalRemoval(metadata); switch (quota_mode) { case QuotaEraseMode::kFull: + case QuotaEraseMode::kHandoff: AbortTenantQuota(tenant_id, metadata.reserved_quota_charge_bytes); ReleaseTenantQuota(tenant_id, metadata.committed_quota_charge_bytes); @@ -2058,6 +3108,10 @@ MasterService::EraseMetadata( AbortTenantQuota(tenant_id, metadata.reserved_quota_charge_bytes); break; } + // CVM 多 submaster：若该对象的句柄由 peer 分配（远程分配/迁移入）， + // 广播释放到所有 peer 的 keepalive 注册表；kHandoff（slot 迁移）跳过， + // 保留数据字节并让导入方接管远程释放责任。 + EnqueueRemoteFreeIfTracked(tenant_id, key, quota_mode); auto next = tenant_state.metadata.erase(it); DecrementTenantMetadataObjectCount(tenant_id); if (had_completed_disk && shard) { @@ -2322,6 +3376,7 @@ auto MasterService::UnmountSegment(const UUID& segment_id, std::string(bytes.begin(), bytes.end())); } RecomputeTenantEffectiveQuotas(); + RemoveSegmentOwnerForCvm(segment_id); return {}; } @@ -2667,6 +3722,31 @@ void MasterService::RestoreFromStandbySnapshot( std::unordered_map> objects_by_shard; + + // P4：晋升时只物化「本机负责 slot」的对象元数据（数据字节留在 segment）。 + // 仅在 etcd HA 动态分区下过滤；非 HA / 单机 / 测试路径 lookup 为空，退化 + // 为恢复全部。ResolveOwnedSlotsForCvm 失败时 sticky 沿用上一轮结果 + // （standby 晋升前为空 → 全量恢复），同样等价于不过滤。 + std::vector owned_slot_lookup; + { + const bool kv_partition_enabled = + enable_ha_ && ha_backend_type_ == "etcd" && + !master_id_.empty() && !cluster_id_.empty(); + if (kv_partition_enabled) { + const std::vector owned_slots = + ResolveOwnedSlotsForCvm(); + if (!owned_slots.empty()) { + owned_slot_lookup.assign(cvm::kSlotCount, false); + for (uint16_t slot : owned_slots) { + owned_slot_lookup[slot] = true; + } + // 同步到读路径 owned-slot 位图（A1），让新 primary 立即按 + // 最新分区拒绝非本机 slot 的读请求。 + UpdateOwnedSlots(owned_slots); + } + } + } + for (const auto& entry : objects) { auto [tenant_id, user_key] = resolve_standby_object(entry); if (!tenant_id.IsValid()) { @@ -2675,6 +3755,13 @@ void MasterService::RestoreFromStandbySnapshot( << ", skipping"; continue; } + // slot 过滤：只物化本机负责 slot 的元数据（P4 元数据迁移）。 + if (!owned_slot_lookup.empty()) { + const uint16_t slot = cvm::KeySlot(tenant_id, user_key); + if (!owned_slot_lookup[slot]) { + continue; + } + } const auto shard_idx = entry.metadata.group_id.empty() ? getShardIndex(tenant_id, user_key) : getShardIndex(entry.metadata.group_id); @@ -3180,9 +4267,44 @@ auto MasterService::GetOffloadEndpoints() auto MasterService::GetReplicaList(const std::string& key, const TenantId& tenant_id) -> tl::expected { - std::shared_lock shared_lock(snapshot_mutex_); const auto object_id = MakeObjectIdentityForRequest(key, tenant_id); + // A1：读路径 slot 所有权校验。分区重均衡后本机不再负责该 slot 时，先 + // 尝试向 slot owner 转发（方案 B 第二阶段）；转发失败再回退到 + // SLOT_NOT_OWNED 让客户端根据最新 slot→master 视图重新路由。 + const uint16_t slot = cvm::KeySlot(object_id.tenant_id, object_id.user_key); + if (!OwnsSlot(slot)) { +#ifdef STORE_USE_ETCD + auto owner = ResolveSlotOwnerMasterId(slot); + if (owner && !owner->empty() && *owner != master_id_ && + inter_master_rpc_) { + auto result = inter_master_rpc_->GetReplicaList( + *owner, key, object_id.tenant_id.value()); + if (result.has_value()) { + LOG(INFO) << "GetReplicaList forwarded: key=" << key + << " slot=" << slot << " -> owner=" << *owner; + return result; + } + VLOG(1) << "GetReplicaList forward failed: key=" << key + << " owner=" << *owner + << " error=" << toString(result.error()); + } +#endif + LOG(WARNING) << "GetReplicaList: key=" << key << " slot=" << slot + << " not owned by this master and forwarding failed, " + "rejecting with SLOT_NOT_OWNED (client should " + "re-route)"; + return tl::make_unexpected(ErrorCode::SLOT_NOT_OWNED); + } + + return GetReplicaListLocal(object_id); +} + +auto MasterService::GetReplicaListLocal(const ObjectIdentity& object_id) + -> tl::expected { + std::shared_lock shared_lock(snapshot_mutex_); + const std::string& key = object_id.user_key; + GetReplicaListResponse resp({}, default_kv_lease_ttl_); bool promotion_eligible = false; { @@ -3314,6 +4436,79 @@ MasterService::BatchGetReplicaList(const std::vector& keys, return results; } + const TenantId& normalized_tenant = ResolveRequestTenantId(tenant_id); + + // A1 + 方案 B 第二阶段：先按 slot 所有权把 keys 拆成「本地组」与 + // 「转发组（按 owner master_id 分组）」。本地组走本地批量查询，转发组 + // 通过 InterMasterRpcClient 转发给对应 slot owner。 + std::vector local_indices; + std::map> forward_groups; + for (size_t i = 0; i < keys.size(); ++i) { + const uint16_t slot = cvm::KeySlot(normalized_tenant, keys[i]); + if (OwnsSlot(slot)) { + local_indices.push_back(i); + continue; + } +#ifdef STORE_USE_ETCD + auto owner = ResolveSlotOwnerMasterId(slot); + if (owner && !owner->empty() && *owner != master_id_ && + inter_master_rpc_) { + forward_groups[*owner].push_back(i); + continue; + } +#endif + LOG(WARNING) << "BatchGetReplicaList: key=" << keys[i] + << " slot=" << slot + << " not owned by this master and no forward target, " + "rejecting with SLOT_NOT_OWNED"; + results[i] = tl::make_unexpected(ErrorCode::SLOT_NOT_OWNED); + } + + if (!local_indices.empty()) { + std::vector local_keys; + local_keys.reserve(local_indices.size()); + for (size_t idx : local_indices) { + local_keys.push_back(keys[idx]); + } + auto local_results = + BatchGetReplicaListLocal(local_keys, normalized_tenant); + for (size_t j = 0; j < local_indices.size(); ++j) { + results[local_indices[j]] = std::move(local_results[j]); + } + } + +#ifdef STORE_USE_ETCD + for (const auto& [owner, indices] : forward_groups) { + std::vector group_keys; + group_keys.reserve(indices.size()); + for (size_t idx : indices) { + group_keys.push_back(keys[idx]); + } + auto group_result = inter_master_rpc_->BatchGetReplicaList( + owner, group_keys, normalized_tenant.value()); + for (size_t j = 0; j < indices.size() && j < group_result.size(); + ++j) { + results[indices[j]] = std::move(group_result[j]); + } + } +#endif + + return results; +} + +std::vector> +MasterService::BatchGetReplicaListLocal(const std::vector& keys, + const TenantId& tenant_id) { + using GetResult = tl::expected; + + assert(tenant_id.IsValid()); + + std::vector results( + keys.size(), tl::make_unexpected(ErrorCode::OBJECT_NOT_FOUND)); + if (keys.empty()) { + return results; + } + const TenantId& normalized_tenant = ResolveRequestTenantId(tenant_id); constexpr size_t kInvalidKeyIndex = std::numeric_limits::max(); std::array key_list_heads; @@ -3352,6 +4547,7 @@ MasterService::BatchGetReplicaList(const std::vector& keys, original_idx != kInvalidKeyIndex; original_idx = next_key_indexes[original_idx]) { const std::string& key = keys[original_idx]; + MasterMetricManager::instance().inc_total_get_nums(); if (tenant_it == shard->tenants.end()) { @@ -3565,84 +4761,110 @@ auto MasterService::AllocateAndInsertMetadata( : config.host_id; } - ScopedAllocatorAccess allocator_access = - segment_manager_.getAllocatorAccess(); - const auto& allocator_manager = allocator_access.getAllocatorManager(); - if (allocator_manager.getNames().size() >= config.replica_num) { - for (const auto& name : allocator_manager.getNames()) { - const auto* allocators = allocator_manager.getAllocators(name); - if (allocators != nullptr && - std::any_of(allocators->begin(), allocators->end(), - [](const auto& allocator) { - return allocator && allocator->size() > 0; - })) { - memory_eviction_may_help = true; - break; + std::vector preferred_segments; + tl::expected, ErrorCode> allocation_result = + tl::make_unexpected(ErrorCode::NO_AVAILABLE_HANDLE); + { + ScopedAllocatorAccess allocator_access = + segment_manager_.getAllocatorAccess(); + const auto& allocator_manager = + allocator_access.getAllocatorManager(); + if (allocator_manager.getNames().size() >= config.replica_num) { + for (const auto& name : allocator_manager.getNames()) { + const auto* allocators = + allocator_manager.getAllocators(name); + if (allocators != nullptr && + std::any_of(allocators->begin(), allocators->end(), + [](const auto& allocator) { + return allocator && + allocator->size() > 0; + })) { + memory_eviction_may_help = true; + break; + } } } - } - std::vector preferred_segments; - auto append_preferred_segment = [&preferred_segments]( - const std::string& segment_name) { - if (!segment_name.empty() && - std::find(preferred_segments.begin(), preferred_segments.end(), - segment_name) == preferred_segments.end()) { - preferred_segments.push_back(segment_name); - } - }; - if (!config.preferred_segment.empty()) { - append_preferred_segment(config.preferred_segment); - } else { - for (const auto& preferred_segment : config.preferred_segments) { - append_preferred_segment(preferred_segment); - } - } - if (!writer_host_id.empty()) { - auto host_ordered_segments = - allocator_access.GetHostOrderedSegments(writer_host_id, key); - for (const auto& segment_name : host_ordered_segments) { - append_preferred_segment(segment_name); + auto append_preferred_segment = [&preferred_segments]( + const std::string& + segment_name) { + if (!segment_name.empty() && + std::find(preferred_segments.begin(), + preferred_segments.end(), + segment_name) == preferred_segments.end()) { + preferred_segments.push_back(segment_name); + } + }; + if (!config.preferred_segment.empty()) { + append_preferred_segment(config.preferred_segment); + } else { + for (const auto& preferred_segment : + config.preferred_segments) { + append_preferred_segment(preferred_segment); + } } - if (!host_ordered_segments.empty()) { - VLOG(1) << "key=" << key - << ", writer_host_id=" << writer_host_id - << ", local_first_preferred_segments=" - << host_ordered_segments.size(); + if (!writer_host_id.empty()) { + auto host_ordered_segments = + allocator_access.GetHostOrderedSegments(writer_host_id, + key); + for (const auto& segment_name : host_ordered_segments) { + append_preferred_segment(segment_name); + } + if (!host_ordered_segments.empty()) { + VLOG(1) << "key=" << key + << ", writer_host_id=" << writer_host_id + << ", local_first_preferred_segments=" + << host_ordered_segments.size(); + } } - } - const SsdMetricsProvider* ssd_provider = nullptr; - std::optional ssd_access; - if (allocation_strategy_type_ == - AllocationStrategyType::SSD_FREE_RATIO_FIRST) { - ssd_access.emplace(segment_manager_.getLocalDiskSegmentAccess()); - ssd_provider = &*ssd_access; - } + const SsdMetricsProvider* ssd_provider = nullptr; + std::optional ssd_access; + if (allocation_strategy_type_ == + AllocationStrategyType::SSD_FREE_RATIO_FIRST) { + ssd_access.emplace( + segment_manager_.getLocalDiskSegmentAccess()); + ssd_provider = &*ssd_access; + } - SpDiag::PerfPoint pt_alloc_mem(PerfKey::MASTER_PUT_ALLOCATE_MEM, - SpDiag::PerfLevel::KEY_MODULE); - pt_alloc_mem.Start(); - auto allocation_result = allocation_strategy_->Allocate( - allocator_manager, value_length, config.replica_num, - preferred_segments, std::set(), ReplicaType::MEMORY, - ssd_provider); - pt_alloc_mem.End(allocation_result.has_value() ? 0 : -1); + SpDiag::PerfPoint pt_alloc_mem(PerfKey::MASTER_PUT_ALLOCATE_MEM, + SpDiag::PerfLevel::KEY_MODULE); + pt_alloc_mem.Start(); + allocation_result = allocation_strategy_->Allocate( + allocator_manager, value_length, config.replica_num, + preferred_segments, std::set(), + ReplicaType::MEMORY, ssd_provider); + pt_alloc_mem.End(allocation_result.has_value() ? 0 : -1); + } // allocator_access 在此释放；远程转发在锁外执行，避免分布式死锁 if (!allocation_result.has_value()) { - VLOG(1) << "Failed to allocate replicas for key=" << key + VLOG(1) << "Failed to allocate replicas locally for key=" << key << ", error: " << allocation_result.error(); - if (allocation_result.error() == ErrorCode::INVALID_PARAMS) { - abort_reserved_quota(); - return tl::make_unexpected(ErrorCode::INVALID_PARAMS); - } - if (write_mode != ReplicaWriteMode::FLEXIBLE_DUAL_REPLICA) { - MasterMetricManager::instance().inc_put_start_alloc_failures(); - if (memory_eviction_may_help) { - need_mem_eviction_ = true; + // CVM 多 submaster（方案 B 第二阶段）：本机本地分配失败（未挂载 + // segment 或空间不足）时，尝试向持有该 segment 的 peer submaster + // 转发分配。远程分配成功则以 dummy-allocator 副本物化到本机元数据， + // 真实句柄留在 segment owner 的 keepalive 注册表中。 + auto remote_replicas = TryAllocateReplicasRemotely( + key, tenant_id, value_length, config.replica_num, + preferred_segments); + if (remote_replicas.has_value()) { + replicas = std::move(remote_replicas.value()); + allocated_memory_replicas = replicas.size(); + } else { + if (allocation_result.error() == ErrorCode::INVALID_PARAMS) { + abort_reserved_quota(); + return tl::make_unexpected(ErrorCode::INVALID_PARAMS); + } + if (write_mode != ReplicaWriteMode::FLEXIBLE_DUAL_REPLICA) { + MasterMetricManager::instance() + .inc_put_start_alloc_failures(); + if (memory_eviction_may_help) { + need_mem_eviction_ = true; + } + abort_reserved_quota(); + return tl::make_unexpected( + ErrorCode::NO_AVAILABLE_HANDLE); } - abort_reserved_quota(); - return tl::make_unexpected(ErrorCode::NO_AVAILABLE_HANDLE); } } else { allocated_memory_replicas = allocation_result->size(); @@ -3819,6 +5041,32 @@ auto MasterService::PutStart(const UUID& client_id, const std::string& key, UpdateClientHostId(client_id, config.host_id); +#ifdef STORE_USE_ETCD + // 写路径 slot 归属校验 + 完整转发（模型 B）：本机若非该 key 的 slot + // owner，不得在本地写入元数据（否则元数据落在"非环 owner"上，而读路径 + // 按环归属，会造成"写后读不到/写到错误位置"的不一致）。此处把整个 + // PutStart 转发给 slot owner，由其本地分配 + 写元数据并返回 Descriptor。 + // owner 侧 peer 互信（OwnsSlot==true）不再二次转发，转发链止于第一跳。 + { + const uint16_t slot = + cvm::KeySlot(object_id.tenant_id, object_id.user_key); + if (!OwnsSlot(slot)) { + auto owner = ResolveSlotOwnerMasterId(slot); + if (owner && !owner->empty() && *owner != master_id_ && + inter_master_rpc_) { + LOG(INFO) << "PutStart forwarded: key=" << object_id.user_key + << " slot=" << slot << " -> owner=" << *owner; + return inter_master_rpc_->PutStart( + *owner, client_id, object_id.user_key, + object_id.tenant_id.value(), slice_length, config); + } + LOG(INFO) << "PutStart rejected with SLOT_NOT_OWNED: key=" + << object_id.user_key << " slot=" << slot; + return tl::make_unexpected(ErrorCode::SLOT_NOT_OWNED); + } + } +#endif + if ((memory_allocator_type_ == BufferAllocatorType::CACHELIB) && (slice_length > kMaxSliceSize)) { LOG(ERROR) << "key=" << key << ", slice_length=" << slice_length @@ -4419,6 +5667,31 @@ auto MasterService::UpsertStart(const UUID& client_id, const std::string& key, UpdateClientHostId(client_id, config.host_id); +#ifdef STORE_USE_ETCD + // 写路径 slot 归属校验 + 完整转发（模型 B）：Upsert 为覆盖式写，一旦 + // client 路由过期把请求发到非 slot owner，错位覆盖比 PutStart 危害更大。 + // 此处与 PutStart 对称：把整个 UpsertStart 转发给 slot owner 执行。 + // owner 侧 peer 互信（OwnsSlot==true）不再二次转发，转发链止于第一跳。 + { + const uint16_t slot = + cvm::KeySlot(object_id.tenant_id, object_id.user_key); + if (!OwnsSlot(slot)) { + auto owner = ResolveSlotOwnerMasterId(slot); + if (owner && !owner->empty() && *owner != master_id_ && + inter_master_rpc_) { + LOG(INFO) << "UpsertStart forwarded: key=" << object_id.user_key + << " slot=" << slot << " -> owner=" << *owner; + return inter_master_rpc_->UpsertStart( + *owner, client_id, object_id.user_key, + object_id.tenant_id.value(), slice_length, config); + } + LOG(INFO) << "UpsertStart rejected with SLOT_NOT_OWNED: key=" + << object_id.user_key << " slot=" << slot; + return tl::make_unexpected(ErrorCode::SLOT_NOT_OWNED); + } + } +#endif + if ((memory_allocator_type_ == BufferAllocatorType::CACHELIB) && (slice_length > kMaxSliceSize)) { LOG(ERROR) << "key=" << key << ", slice_length=" << slice_length @@ -5599,6 +6872,17 @@ auto MasterService::Remove(const std::string& key, const TenantId& tenant_id, } auto& metadata = accessor.Get(); + std::vector local_disk_holders; + metadata.VisitReplicas( + [](const Replica& replica) { + return replica.is_local_disk_replica(); + }, + [&local_disk_holders](Replica& replica) { + auto client_id = replica.get_local_disk_client_id(); + if (client_id.has_value()) { + local_disk_holders.push_back(client_id.value()); + } + }); if (!force && !metadata.IsLeaseExpired()) { VLOG(1) << "key=" << key << ", error=object_has_lease"; @@ -5636,10 +6920,15 @@ auto MasterService::Remove(const std::string& key, const TenantId& tenant_id, auto persist_result = AppendReservedOpLogWithDurableFinalize( std::move(reservation.value()), OpType::REMOVE, object_id.tenant_id.value(), key, {}, - [this, removed_ids = std::move(removed_ids)]( + [this, removed_ids = std::move(removed_ids), + local_disk_holders, + tenant_id_for_task = object_id.tenant_id.value(), key]( const OpLogEntry& durable_entry) { FinalizeRemovedReplicasAfterDurable( durable_entry, removed_ids, QuotaEraseMode::kFull); + EnqueueRemoveTasks( + local_disk_holders, + RemoveTaskItem{tenant_id_for_task, key}); }); if (!persist_result) { return tl::make_unexpected(persist_result.error()); @@ -5648,7 +6937,14 @@ auto MasterService::Remove(const std::string& key, const TenantId& tenant_id, } } PublishKvRemoved(key, metadata, object_id.tenant_id); + + // Before erasing metadata, collect LOCAL_DISK replica holders so we + // can notify them to reclaim SSD space via RemoveObjectHeartbeat. accessor.Erase(); + + // Push removed key to each LOCAL_DISK holder's removed_keys queue. + EnqueueRemoveTasks(local_disk_holders, RemoveTaskItem{tenant_id.value(), key}); + return {}; } @@ -6045,6 +7341,18 @@ auto MasterService::BatchRemove(const std::vector& keys, auto& metadata = it->second; + std::vector batch_local_disk_holders; + metadata.VisitReplicas( + [](const Replica& replica) { + return replica.is_local_disk_replica(); + }, + [&batch_local_disk_holders](Replica& replica) { + auto cid = replica.get_local_disk_client_id(); + if (cid.has_value()) { + batch_local_disk_holders.push_back(cid.value()); + } + }); + if (!force && !metadata.IsLeaseExpired(now)) { VLOG(1) << "key=" << key << ", error=object_has_lease"; results[original_idx] = @@ -6087,11 +7395,16 @@ auto MasterService::BatchRemove(const std::vector& keys, AppendReservedOpLogWithDurableFinalize( std::move(reservation.value()), OpType::REMOVE, normalized_tenant.value(), key, {}, - [this, removed_ids = std::move(removed_ids)]( + [this, removed_ids = std::move(removed_ids), + batch_local_disk_holders, + tenant_id = normalized_tenant.value(), key]( const OpLogEntry& durable_entry) { FinalizeRemovedReplicasAfterDurable( durable_entry, removed_ids, QuotaEraseMode::kFull); + EnqueueRemoveTasks( + batch_local_disk_holders, + RemoveTaskItem{tenant_id, key}); }); if (!persist_result) { results[original_idx] = @@ -6102,11 +7415,20 @@ auto MasterService::BatchRemove(const std::vector& keys, continue; } } + + // Collect LOCAL_DISK replica holders before erasing, so we + // can notify them to reclaim SSD space via RemoveObjectHeartbeat. EraseMetadata(tenant_state, it, normalized_tenant, QuotaEraseMode::kFull, &shard); if (tenant_state.Empty()) { shard->tenants.erase(tenant_it); } + + // Push removed key to each LOCAL_DISK holder's removed_keys queue. + EnqueueRemoveTasks( + batch_local_disk_holders, + RemoveTaskItem{normalized_tenant.value(), key}); + results[original_idx] = {}; // Success } } @@ -6331,6 +7653,64 @@ auto MasterService::PollRemoveAll(const UUID& client_id) return result; } +auto MasterService::RemoveObjectHeartbeat(const UUID& client_id) + -> tl::expected, ErrorCode> { + std::shared_lock shared_lock(snapshot_mutex_); + ScopedLocalDiskSegmentAccess local_disk_segment_access = + segment_manager_.getLocalDiskSegmentAccess(); + auto& client_local_disk_segment = + local_disk_segment_access.getClientLocalDiskSegment(); + auto local_disk_segment_it = client_local_disk_segment.find(client_id); + if (local_disk_segment_it == client_local_disk_segment.end()) { + return tl::make_unexpected(ErrorCode::SEGMENT_NOT_FOUND); + } + { + MutexLocker locker(&local_disk_segment_it->second->offloading_mutex_); + return local_disk_segment_it->second->removed_keys; + } +} + +void MasterService::EnqueueRemoveTasks( + const std::vector& holder_ids, const RemoveTaskItem& task) { + if (holder_ids.empty()) return; + ScopedLocalDiskSegmentAccess access = + segment_manager_.getLocalDiskSegmentAccess(); + auto& segments = access.getClientLocalDiskSegment(); + for (const auto& holder_id : holder_ids) { + auto it = segments.find(holder_id); + if (it == segments.end()) continue; + MutexLocker locker(&it->second->offloading_mutex_); + if (std::find(it->second->removed_keys.begin(), + it->second->removed_keys.end(), task) == + it->second->removed_keys.end()) { + it->second->removed_keys.push_back(task); + } + } +} + +auto MasterService::AckRemoveObjectHeartbeat( + const UUID& client_id, const std::vector& tasks) + -> tl::expected { + std::shared_lock shared_lock(snapshot_mutex_); + ScopedLocalDiskSegmentAccess access = + segment_manager_.getLocalDiskSegmentAccess(); + auto& segments = access.getClientLocalDiskSegment(); + auto it = segments.find(client_id); + if (it == segments.end()) { + return tl::make_unexpected(ErrorCode::SEGMENT_NOT_FOUND); + } + MutexLocker locker(&it->second->offloading_mutex_); + auto& pending = it->second->removed_keys; + pending.erase(std::remove_if(pending.begin(), pending.end(), + [&tasks](const RemoveTaskItem& task) { + return std::find(tasks.begin(), + tasks.end(), task) != + tasks.end(); + }), + pending.end()); + return {}; +} + auto MasterService::ReportSsdCapacity(const UUID& client_id, int64_t ssd_total_capacity_bytes) -> tl::expected { @@ -11207,7 +12587,8 @@ ErrorCode MasterService::InitializeBatchOpLogWriter( return ErrorCode::INVALID_PARAMS; } - auto storage = std::make_unique(cluster_id_, *backend); + auto storage = std::make_unique(cluster_id_, *backend, + master_id_); DurablePrefix durable_prefix; ErrorCode err = storage->InitDurablePrefix(durable_prefix); if (err != ErrorCode::OK) { diff --git a/mooncake-store/src/partition/partition_router.cpp b/mooncake-store/src/partition/partition_router.cpp new file mode 100644 index 0000000000..488230eabd --- /dev/null +++ b/mooncake-store/src/partition/partition_router.cpp @@ -0,0 +1,94 @@ +#include "partition/partition_router.h" + +#include + +#include + +#include "cvm/cvm_keys.h" +#include "etcd_helper.h" +#include "partition/kv_hash_map.h" +#include "ylt/struct_json/json_reader.h" + +namespace mooncake { +namespace partition { + +void PartitionRouter::LoadSlotOwners( + const std::vector& owners) { + std::unordered_map next; + next.reserve(owners.size()); + for (const auto& owner : owners) { + if (!owner.primary_master_id.empty() && + owner.state == static_cast(cvm::SlotState::kStable)) { + next[owner.slot] = owner.primary_master_id; + } + } + + const size_t valid = next.size(); + { + SharedMutexLocker locker(&mutex_); + slot_to_submaster_ = std::move(next); + } + LOG(INFO) << "PartitionRouter loaded " << valid << " slot->submaster" + << " entries (input " << owners.size() << " SlotOwner records)"; +} + +ErrorCode PartitionRouter::LoadFromEtcdSnapshot( + const std::string& cluster_namespace) { + const std::string key = cvm::KvViewSnapshotKey(cluster_namespace); + std::string value; + EtcdRevisionId revision = 0; + ErrorCode err = EtcdHelper::Get(key.data(), key.size(), value, revision); + if (err != ErrorCode::OK) { + LOG(WARNING) << "PartitionRouter read snapshot failed: " << key + << " err=" << err; + return err; + } + + cvm::KvViewSnapshot snapshot; + try { + struct_json::from_json(snapshot, value); + } catch (const std::exception& e) { + LOG(ERROR) << "PartitionRouter deserialize snapshot failed: " + << e.what(); + return ErrorCode::DESERIALIZE_FAIL; + } + + LoadSlotOwners(snapshot.slot_owners); + LOG(INFO) << "PartitionRouter refreshed from snapshot " << key + << " version=" << snapshot.version; + return ErrorCode::OK; +} + +std::optional PartitionRouter::ResolveSubmaster( + uint16_t slot) const { + SharedMutexLocker locker(&mutex_, shared_lock); + auto it = slot_to_submaster_.find(slot); + if (it == slot_to_submaster_.end()) { + LOG(WARNING) << "PartitionRouter no submaster for slot " << slot; + return std::nullopt; + } + return it->second; +} + +std::optional PartitionRouter::Route( + const TenantId& tenant, const std::string& key) const { + return ResolveSubmaster(KvHashMap::Compute(tenant, key)); +} + +void PartitionRouter::Clear() { + size_t old_size = 0; + { + SharedMutexLocker locker(&mutex_); + old_size = slot_to_submaster_.size(); + slot_to_submaster_.clear(); + } + LOG(INFO) << "PartitionRouter cleared " << old_size << " entries"; +} + +size_t PartitionRouter::Size() const { + SharedMutexLocker locker(&mutex_, shared_lock); + return slot_to_submaster_.size(); +} + +} // namespace partition +} // namespace mooncake diff --git a/mooncake-store/src/real_client.cpp b/mooncake-store/src/real_client.cpp index c7eb5df178..5e5f5bed8c 100644 --- a/mooncake-store/src/real_client.cpp +++ b/mooncake-store/src/real_client.cpp @@ -2397,6 +2397,7 @@ tl::expected RealClient::remove_internal( if (!remove_result) { return tl::unexpected(remove_result.error()); } + // SSD tombstone is handled by the storage node via RemoveObjectHeartbeat. return {}; } @@ -2436,7 +2437,9 @@ std::vector> RealClient::batchRemove_internal( return std::vector>( keys.size(), tl::unexpected(ErrorCode::INVALID_PARAMS)); } - return client_->BatchRemove(keys, force); + auto results = client_->BatchRemove(keys, force); + // SSD tombstone is handled by the storage node via RemoveObjectHeartbeat. + return results; } std::vector RealClient::batchRemove(const std::vector &keys, diff --git a/mooncake-store/src/rpc_service.cpp b/mooncake-store/src/rpc_service.cpp index e3b7891086..11fe6f60c4 100644 --- a/mooncake-store/src/rpc_service.cpp +++ b/mooncake-store/src/rpc_service.cpp @@ -8,6 +8,7 @@ #include #include +#include "common.h" #include "ha_metric_manager.h" #include "master_admin_service.h" #include "master_metric_manager.h" @@ -546,6 +547,81 @@ tl::expected WrappedMasterService::PutEnd( return result; } +tl::expected +WrappedMasterService::VChunkPutStart(const std::string& tenant_id, + const std::string& key, + uint64_t total_size, int64_t now_ms) { + return WithRequestTenant( + tenant_id, + [&](const TenantId& resolved_tenant_id) { + return master_service_.VChunkPutStart(resolved_tenant_id, key, + total_size, false, now_ms); + }); +} + +tl::expected WrappedMasterService::VChunkPutEnd( + const std::string& tenant_id, const std::string& key, + const std::string& vchunk_id, int64_t now_ms) { + return WithRequestTenant( + tenant_id, + [&](const TenantId& resolved_tenant_id) { + const auto error = master_service_.VChunkPutEnd( + resolved_tenant_id, key, vchunk_id, now_ms); + return error == ErrorCode::OK + ? tl::expected{} + : tl::expected{tl::unexpected(error)}; + }); +} + +tl::expected WrappedMasterService::VChunkPutRevoke( + const std::string& tenant_id, const std::string& key, + const std::string& vchunk_id) { + return WithRequestTenant( + tenant_id, + [&](const TenantId& resolved_tenant_id) { + const auto error = master_service_.VChunkPutRevoke( + resolved_tenant_id, key, vchunk_id); + return error == ErrorCode::OK + ? tl::expected{} + : tl::expected{tl::unexpected(error)}; + }); +} + +tl::expected WrappedMasterService::GetVChunk( + const std::string& tenant_id, const std::string& key) { + return WithRequestTenant( + tenant_id, + [&](const TenantId& resolved_tenant_id) { + return master_service_.AcquireVChunkReadLease( + resolved_tenant_id, key, getCurrentTimeInMilli()); + }); +} + +tl::expected WrappedMasterService::ReleaseVChunkReadLease( + const std::string& lease_id) { + const auto error = master_service_.ReleaseVChunkReadLease(lease_id); + return error == ErrorCode::OK + ? tl::expected{} + : tl::expected{tl::unexpected(error)}; +} + +tl::expected WrappedMasterService::RemoveVChunk( + const std::string& tenant_id, const std::string& key, int64_t now_ms) { + return WithRequestTenant( + tenant_id, + [&](const TenantId& resolved_tenant_id) { + const auto error = master_service_.RemoveVChunk( + resolved_tenant_id, key, now_ms); + return error == ErrorCode::OK + ? tl::expected{} + : tl::expected{tl::unexpected(error)}; + }); +} + +VChunkRuntimeInfo WrappedMasterService::GetVChunkRuntimeInfo() { + return master_service_.GetVChunkRuntimeInfo(); +} + tl::expected WrappedMasterService::PutRevoke( const UUID& client_id, const std::string& key, ReplicaType replica_type, const std::string& tenant_id) { @@ -1728,6 +1804,21 @@ WrappedMasterService::PromotionObjectHeartbeat(const UUID& client_id) { return master_service_.PromotionObjectHeartbeat(client_id); } +tl::expected, ErrorCode> +WrappedMasterService::RemoveObjectHeartbeat(const UUID& client_id) { + ScopedVLogTimer timer(1, "RemoveObjectHeartbeat"); + timer.LogRequest("action=remove_heartbeat"); + return master_service_.RemoveObjectHeartbeat(client_id); +} + +tl::expected +WrappedMasterService::AckRemoveObjectHeartbeat( + const UUID& client_id, const std::vector& tasks) { + ScopedVLogTimer timer(1, "AckRemoveObjectHeartbeat"); + timer.LogRequest("action=ack_remove_heartbeat"); + return master_service_.AckRemoveObjectHeartbeat(client_id, tasks); +} + tl::expected WrappedMasterService::PromotionAllocStart( const UUID& client_id, const std::string& key, const std::string& tenant_id, @@ -1817,11 +1908,152 @@ void WrappedMasterService::RestoreFromStandby( objects, initial_oplog_sequence_id, segments); } +void WrappedMasterService::SetCvmLeaseId(EtcdLeaseId lease_id) { + master_service_.SetCvmLeaseId(lease_id); +} + +ErrorCode WrappedMasterService::StartSlotOwnerHeartbeat() { + return master_service_.StartSlotOwnerHeartbeat(); +} + +void WrappedMasterService::StopSlotOwnerHeartbeat() { + master_service_.StopSlotOwnerHeartbeat(); +} + +ErrorCode WrappedMasterService::StartInterMasterRpc() { + return master_service_.StartInterMasterRpc(); +} + +void WrappedMasterService::StopInterMasterRpc() { + master_service_.StopInterMasterRpc(); +} + +tl::expected +WrappedMasterService::InterMasterHandshake() { + return execute_rpc( + "InterMasterHandshake", + [&]() -> tl::expected { + InterMasterHandshakeResponse resp; + resp.master_id = master_service_.master_id(); + resp.lease_id = master_service_.cvm_lease_id(); + resp.owned_slot_count = master_service_.GetOwnedSlotCount(); + resp.version = GetMooncakeStoreVersion(); + return resp; + }, + [&](auto& timer) { + timer.LogRequest("self=", master_service_.master_id()); + }, + [] {}, [] {}); +} + +tl::expected, ErrorCode> +WrappedMasterService::InterMasterAllocateReplicas( + const std::string& tenant_id, const std::string& key, + uint64_t slice_length, uint64_t replica_num, + const std::vector& preferred_segments) { + return execute_rpc( + "InterMasterAllocateReplicas", + [&] { + return master_service_.InterMasterAllocateReplicas( + tenant_id, key, slice_length, replica_num, preferred_segments); + }, + [&](auto& timer) { + timer.LogRequest("key=", key, ", slice_length=", slice_length, + ", replica_num=", replica_num); + }, + [] {}, [] {}); +} + +tl::expected WrappedMasterService::InterMasterFreeReplicas( + const std::string& tenant_id, const std::string& key) { + return execute_rpc( + "InterMasterFreeReplicas", + [&] { return master_service_.InterMasterFreeReplicas(tenant_id, key); }, + [&](auto& timer) { timer.LogRequest("key=", key); }, [] {}, [] {}); +} + +tl::expected +WrappedMasterService::InterMasterGetReplicaList(const std::string& key, + const std::string& tenant_id) { + return execute_rpc( + "InterMasterGetReplicaList", + [&] { + return master_service_.InterMasterGetReplicaList(key, tenant_id); + }, + [&](auto& timer) { timer.LogRequest("key=", key); }, [] {}, [] {}); +} + +std::vector> +WrappedMasterService::InterMasterBatchGetReplicaList( + const std::vector& keys, const std::string& tenant_id) { + ScopedVLogTimer timer(1, "InterMasterBatchGetReplicaList"); + timer.LogRequest("keys_count=", keys.size()); + return master_service_.InterMasterBatchGetReplicaList(keys, tenant_id); +} + +tl::expected, ErrorCode> +WrappedMasterService::InterMasterPutStart( + const UUID& client_id, const std::string& key, const std::string& tenant_id, + uint64_t slice_length, const ReplicateConfig& config) { + return execute_rpc( + "InterMasterPutStart", PerfKey::MASTER_RPC_PUT_START, + [&] { + return master_service_.InterMasterPutStart( + client_id, key, tenant_id, slice_length, config); + }, + [&](auto& timer) { + timer.LogRequest("client_id=", client_id, ", key=", key, + ", slice_length=", slice_length); + }, + [] {}, [] {}); +} + +tl::expected, ErrorCode> +WrappedMasterService::InterMasterUpsertStart( + const UUID& client_id, const std::string& key, const std::string& tenant_id, + uint64_t slice_length, const ReplicateConfig& config) { + return execute_rpc( + "InterMasterUpsertStart", PerfKey::MASTER_RPC_UPSERT_START, + [&] { + return master_service_.InterMasterUpsertStart( + client_id, key, tenant_id, slice_length, config); + }, + [&](auto& timer) { + timer.LogRequest("client_id=", client_id, ", key=", key, + ", slice_length=", slice_length); + }, + [] {}, [] {}); +} + void RegisterRpcService( coro_rpc::coro_rpc_server& server, mooncake::WrappedMasterService& wrapped_master_service) { server.register_handler<&mooncake::WrappedMasterService::ExistKey>( &wrapped_master_service); + // Inter-master handshake (CVM multi-submaster coordination). + server.register_handler< + &mooncake::WrappedMasterService::InterMasterHandshake>( + &wrapped_master_service); + // Inter-master allocation forwarding (CVM plan B phase 2). + server.register_handler< + &mooncake::WrappedMasterService::InterMasterAllocateReplicas>( + &wrapped_master_service); + server.register_handler< + &mooncake::WrappedMasterService::InterMasterFreeReplicas>( + &wrapped_master_service); + // Inter-master read forwarding (CVM plan B phase 2). + server.register_handler< + &mooncake::WrappedMasterService::InterMasterGetReplicaList>( + &wrapped_master_service); + server.register_handler< + &mooncake::WrappedMasterService::InterMasterBatchGetReplicaList>( + &wrapped_master_service); + // Inter-master write forwarding (model B). + server.register_handler<&mooncake::WrappedMasterService::InterMasterPutStart>( + &wrapped_master_service); + server.register_handler< + &mooncake::WrappedMasterService::InterMasterUpsertStart>( + &wrapped_master_service); server.register_handler<&mooncake::WrappedMasterService::BatchQueryIp>( &wrapped_master_service); server.register_handler<&mooncake::WrappedMasterService::BatchReplicaClear>( @@ -1840,6 +2072,22 @@ void RegisterRpcService( &wrapped_master_service); server.register_handler<&mooncake::WrappedMasterService::PutRevoke>( &wrapped_master_service); + server.register_handler<&mooncake::WrappedMasterService::VChunkPutStart>( + &wrapped_master_service); + server.register_handler<&mooncake::WrappedMasterService::VChunkPutEnd>( + &wrapped_master_service); + server.register_handler<&mooncake::WrappedMasterService::VChunkPutRevoke>( + &wrapped_master_service); + server.register_handler<&mooncake::WrappedMasterService::GetVChunk>( + &wrapped_master_service); + server.register_handler< + &mooncake::WrappedMasterService::ReleaseVChunkReadLease>( + &wrapped_master_service); + server.register_handler<&mooncake::WrappedMasterService::RemoveVChunk>( + &wrapped_master_service); + server.register_handler< + &mooncake::WrappedMasterService::GetVChunkRuntimeInfo>( + &wrapped_master_service); server.register_handler<&mooncake::WrappedMasterService::BatchPutStart>( &wrapped_master_service); server.register_handler<&mooncake::WrappedMasterService::BatchPutEnd>( @@ -1922,6 +2170,12 @@ void RegisterRpcService( server.register_handler< &mooncake::WrappedMasterService::PromotionObjectHeartbeat>( &wrapped_master_service); + server.register_handler< + &mooncake::WrappedMasterService::RemoveObjectHeartbeat>( + &wrapped_master_service); + server.register_handler< + &mooncake::WrappedMasterService::AckRemoveObjectHeartbeat>( + &wrapped_master_service); server .register_handler<&mooncake::WrappedMasterService::PromotionAllocStart>( &wrapped_master_service); diff --git a/mooncake-store/src/storage_backend.cpp b/mooncake-store/src/storage_backend.cpp index 0084902d39..ab2723b897 100644 --- a/mooncake-store/src/storage_backend.cpp +++ b/mooncake-store/src/storage_backend.cpp @@ -14,11 +14,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include @@ -133,6 +135,40 @@ BucketBackendConfig BucketBackendConfig::FromEnvironment() { config.disable_ssd_eviction = GetEnvOr("MOONCAKE_OFFLOAD_DISABLE_SSD_EVICTION", false); + config.gc_enable = GetEnvOr("MOONCAKE_OFFLOAD_BUCKET_GC_ENABLE", + config.gc_enable); + config.gc_interval_ms = + GetEnvOr("MOONCAKE_OFFLOAD_BUCKET_GC_INTERVAL_MS", + config.gc_interval_ms); + // Parse doubles manually: GetEnvOr uses std::stoll which cannot parse + // fractional values like "0.25". + { + const char* ratio_env = + std::getenv("MOONCAKE_OFFLOAD_BUCKET_GC_DELETED_RATIO"); + if (ratio_env && !std::string(ratio_env).empty()) { + try { + config.gc_deleted_ratio = std::stod(std::string(ratio_env)); + } catch (...) { + // keep default + } + } + const char* wm_env = std::getenv( + "MOONCAKE_OFFLOAD_BUCKET_GC_HIGH_WATERMARK_RATIO"); + if (wm_env && !std::string(wm_env).empty()) { + try { + config.gc_high_watermark_ratio = + std::stod(std::string(wm_env)); + } catch (...) { + // keep default + } + } + } + config.gc_max_buckets_per_round = GetEnvOr( + "MOONCAKE_OFFLOAD_BUCKET_GC_MAX_BUCKETS_PER_ROUND", + config.gc_max_buckets_per_round); + config.gc_merge_enable = GetEnvOr( + "MOONCAKE_OFFLOAD_BUCKET_GC_MERGE_ENABLE", config.gc_merge_enable); + return config; } @@ -1739,6 +1775,12 @@ BucketStorageBackend::BucketStorageBackend( } BucketStorageBackend::~BucketStorageBackend() { + // Stop background GC thread first, before clearing any state it touches. + if (gc_running_.load(std::memory_order_acquire)) { + gc_running_.store(false, std::memory_order_release); + gc_cv_.notify_all(); + if (gc_thread_.joinable()) gc_thread_.join(); + } // Clear file cache to release UringFile instances before destruction // This ensures orderly cleanup of io_uring resources ClearFileCache(); @@ -2068,8 +2110,8 @@ tl::expected BucketStorageBackend::BatchLoad( // Calculate aligned read range int64_t aligned_offset = align_down(actual_offset, kDirectIOAlignment); - int64_t data_end = - actual_offset + static_cast(plan.dest_slice.size); + int64_t data_end = actual_offset + + static_cast(plan.dest_slice.size); int64_t aligned_end = static_cast(align_up( static_cast(data_end), kDirectIOAlignment)); size_t aligned_size = @@ -2095,15 +2137,14 @@ tl::expected BucketStorageBackend::BatchLoad( } } else #endif - { - // Fallback to vector_read for non-UringFile - iovec iov{plan.dest_slice.ptr, plan.dest_slice.size}; - SpDiag::PerfPoint pt_posix(PerfKey::GET_SSD_OWNER_LOAD_POSIX, - SpDiag::PerfLevel::MODULE); - pt_posix.Start(); - read_res = file->vector_read(&iov, 1, actual_offset); - pt_posix.End(read_res ? 0 : -1); - } + { + // Fallback to per-key vector_read for non-UringFile (PosixFile). + iovec iov{plan.dest_slice.ptr, plan.dest_slice.size}; + SpDiag::PerfPoint pt_posix(PerfKey::GET_SSD_OWNER_LOAD_POSIX, + SpDiag::PerfLevel::MODULE); + pt_posix.Start(); + read_res = file->vector_read(&iov, 1, actual_offset); + pt_posix.End(read_res ? 0 : -1); if (stats) { const auto read_us = std::chrono::duration_cast( @@ -2145,6 +2186,7 @@ tl::expected BucketStorageBackend::BatchLoad( return tl::make_unexpected(ErrorCode::FILE_READ_FAIL); } } + } } // bucket_guards go out of scope here, decrementing inflight_reads_ @@ -2292,13 +2334,23 @@ tl::expected BucketStorageBackend::Init() { total_size_ += metadata_it->second->data_size + metadata_it->second->meta_size; for (size_t i = 0; i < metadata_it->second->keys.size(); i++) { + const auto& key = metadata_it->second->keys[i]; + if (std::find(metadata_it->second->tombstones.begin(), + metadata_it->second->tombstones.end(), key) != + metadata_it->second->tombstones.end()) { + metadata_it->second->deleted_bytes_.fetch_add( + metadata_it->second->metadatas[i].key_size + + metadata_it->second->metadatas[i].data_size, + std::memory_order_relaxed); + continue; + } object_bucket_map_.emplace( - metadata_it->second->keys[i], - StorageObjectMetadata{ - metadata_it->first, - metadata_it->second->metadatas[i].offset, - metadata_it->second->metadatas[i].key_size, - metadata_it->second->metadatas[i].data_size, ""}); + key, StorageObjectMetadata{ + metadata_it->first, + metadata_it->second->metadatas[i].offset, + metadata_it->second->metadatas[i].key_size, + metadata_it->second->metadatas[i].data_size, + ""}); } } } @@ -2399,6 +2451,12 @@ tl::expected BucketStorageBackend::Init() { return tl::make_unexpected(ErrorCode::INTERNAL_ERROR); } + // Start background GC thread if enabled. + if (bucket_backend_config_.gc_enable) { + gc_running_.store(true, std::memory_order_release); + gc_thread_ = std::thread(&BucketStorageBackend::GCThreadFunc, this); + } + return {}; } @@ -3432,6 +3490,531 @@ void BucketStorageBackend::RemoveAll() { LOG(INFO) << "RemoveAll: removed " << bucket_ids.size() << " bucket(s)"; } +// --- Explicit-delete-only GC --- +tl::expected BucketStorageBackend::MarkRemoved( + const std::string& key) { + SharedMutexLocker lock(&mutex_); + auto it = object_bucket_map_.find(key); + if (it == object_bucket_map_.end()) { + return {}; + } + int64_t bucket_id = it->second.bucket_id; + int64_t freed = it->second.data_size + it->second.key_size; + auto bucket_it = buckets_.find(bucket_id); + if (bucket_it == buckets_.end()) { + return tl::make_unexpected(ErrorCode::BUCKET_NOT_FOUND); + } + auto bucket = bucket_it->second; + const auto object_metadata = it->second; + object_bucket_map_.erase(it); + bucket->tombstones.push_back(key); + auto persist_result = StoreBucketMetadata(bucket_id, bucket); + if (!persist_result) { + object_bucket_map_.emplace(key, object_metadata); + bucket->tombstones.pop_back(); + return tl::make_unexpected(persist_result.error()); + } + bucket->deleted_bytes_.fetch_add(freed, std::memory_order_relaxed); + ++bucket->generation_; + return {}; +} + +tl::expected BucketStorageBackend::BatchMarkRemoved( + const std::vector& keys) { + for (const auto& key : keys) { + auto result = MarkRemoved(key); + if (!result) { + return result; + } + } + return {}; +} + +bool BucketStorageBackend::CompactBucket(int64_t bucket_id) { + return CompactBuckets({bucket_id}, true); +} + +bool BucketStorageBackend::CompactBuckets( + const std::vector& bucket_ids, bool space_pressure) { + if (bucket_ids.empty()) return true; + + // Step 1: lock-snapshot live keys from ALL old buckets + mark compacting_ + struct LiveKeyInfo { + std::string key; + int64_t old_bucket_id; + BucketObjectMetadata meta; + }; + std::vector live_keys_info; + // old_bucket_id -> {shared_ptr, last_access_ns} + std::unordered_map, int64_t>> + old_buckets; + std::unordered_map old_bucket_generations; + { + SharedMutexLocker lock(&mutex_); + for (int64_t bid : bucket_ids) { + auto it = buckets_.find(bid); + if (it == buckets_.end()) continue; + auto& bucket = it->second; + if (bucket->compacting_.load(std::memory_order_relaxed)) continue; + bucket->compacting_.store(true, std::memory_order_relaxed); + int64_t ts = bucket->last_access_ns_.load( + std::memory_order_relaxed); + old_buckets[bid] = {bucket, ts}; + old_bucket_generations[bid] = bucket->generation_; + + for (size_t i = 0; i < bucket->keys.size(); ++i) { + const auto& key = bucket->keys[i]; + auto map_it = object_bucket_map_.find(key); + if (map_it != object_bucket_map_.end() && + map_it->second.bucket_id == bid) { + live_keys_info.push_back( + {key, bid, bucket->metadatas[i]}); + } + } + } + } + + if (old_buckets.empty()) return true; + + auto reset_compacting = [&]() { + for (auto& [bid, pr] : old_buckets) { + pr.first->compacting_.store(false, std::memory_order_relaxed); + } + }; + + // Identify buckets with zero live keys — delete them immediately + // without waiting for merge. Buckets with live keys participate in + // the merge. + std::set buckets_with_live; + for (const auto& info : live_keys_info) { + buckets_with_live.insert(info.old_bucket_id); + } + + std::vector empty_buckets; + for (auto& [bid, pr] : old_buckets) { + if (buckets_with_live.find(bid) == buckets_with_live.end()) { + empty_buckets.push_back(bid); + } + } + + if (!empty_buckets.empty()) { + std::vector> to_drain; + { + SharedMutexLocker lock(&mutex_); + for (int64_t bid : empty_buckets) { + auto it = buckets_.find(bid); + if (it != buckets_.end()) { + total_size_ -= + it->second->data_size + it->second->meta_size; + buckets_.erase(it); + int64_t ts = old_buckets[bid].second; + lru_index_.erase({ts, bid}); + to_drain.push_back(old_buckets[bid].first); + } + } + } + for (auto& bucket : to_drain) { + WaitForInflightReads(bucket); + } + for (int64_t bid : empty_buckets) { + DeleteBucketFiles(bid); + } + // Remove deleted buckets from old_buckets so they don't interfere + // with the merge logic below. + for (int64_t bid : empty_buckets) { + old_buckets.erase(bid); + } + } + + // If no live keys remain (all buckets were empty), we're done. + if (live_keys_info.empty()) { + return true; + } + + // Step 2: group live keys by bucket_keys_limit / bucket_size_limit + // using metadata only (no file IO). Only the FIRST group that fills up + // will be written as a new bucket; remaining keys are deferred to the + // next round. + struct GroupedKey { + std::string key; + int64_t old_bucket_id; + BucketObjectMetadata meta; + int64_t total_size; // key_size + data_size + }; + std::vector first_group_keys; + int64_t group_count = 0; + int64_t group_size = 0; + + for (const auto& info : live_keys_info) { + int64_t key_total = + info.meta.key_size + info.meta.data_size; + if (group_count >= bucket_backend_config_.bucket_keys_limit || + (group_count > 0 && group_size + key_total > + bucket_backend_config_.bucket_size_limit)) { + break; // first group is full + } + first_group_keys.push_back( + {info.key, info.old_bucket_id, info.meta, key_total}); + group_size += key_total; + ++group_count; + } + + // Check if first group is full enough to write. + bool group_full = + (group_count >= bucket_backend_config_.bucket_keys_limit) || + (group_size >= bucket_backend_config_.bucket_size_limit); + if (!group_full && !space_pressure) { + // Not enough live keys to fill a bucket; defer to next round. + LOG(INFO) << "[GC] CompactBuckets deferred: group_count=" + << group_count + << " group_size=" << group_size + << " bucket_keys_limit=" + << bucket_backend_config_.bucket_keys_limit + << " bucket_size_limit=" + << bucket_backend_config_.bucket_size_limit + << " total_live_keys=" << live_keys_info.size(); + reset_compacting(); + return true; + } + + // Step 3: read ONLY the first group's live key data from old buckets + // (lock-free IO). Open each old bucket file once, read only the keys + // that are in the first group. + std::unordered_map live_data_buffers; + std::unordered_map> first_group; + + // Group first_group_keys by old_bucket_id to open each file once. + std::unordered_map> keys_by_bucket; + for (const auto& gk : first_group_keys) { + keys_by_bucket[gk.old_bucket_id].push_back(&gk); + } + + for (auto& [bid, keys_ptr] : keys_by_bucket) { + auto& old_bucket = old_buckets[bid].first; + bool read_ok = true; + { + BucketReadGuard guard(old_bucket); + auto data_path = GetBucketDataPath(bid); + if (!data_path) { + read_ok = false; + } else { + auto file_result = + OpenFile(data_path.value(), FileMode::Read); + if (!file_result) { + read_ok = false; + } else { + auto& file = file_result.value(); + for (const auto* gk : keys_ptr) { + std::string data; + data.resize(gk->meta.data_size); + int64_t actual_offset = + gk->meta.offset + gk->meta.key_size; + iovec iov{data.data(), + static_cast(gk->meta.data_size)}; + auto read_res = + file->vector_read(&iov, 1, actual_offset); + if (!read_res || + read_res.value() != + static_cast(gk->meta.data_size)) { + LOG(ERROR) + << "CompactBuckets: read failed for key: " + << gk->key << ", bucket_id=" << bid; + read_ok = false; + break; + } + live_data_buffers[gk->key] = std::move(data); + first_group.emplace( + gk->key, + std::vector{Slice{ + live_data_buffers[gk->key].data(), + live_data_buffers[gk->key].size()}}); + } + } + } + } // guard released + + if (!read_ok) { + reset_compacting(); + return false; + } + } + + // Step 4: write the first group as a new bucket. + int64_t new_bucket_id = bucket_id_generator_->NextId(); + std::vector iovs; + std::vector new_metas; + auto build_result = + BuildBucket(new_bucket_id, first_group, iovs, new_metas); + if (!build_result) { + LOG(ERROR) << "CompactBuckets: BuildBucket failed"; + reset_compacting(); + return false; + } + auto write_result = + WriteBucket(new_bucket_id, build_result.value(), iovs); + if (!write_result) { + LOG(ERROR) << "CompactBuckets: WriteBucket failed"; + reset_compacting(); + return false; + } + + // Step 5: atomic swap under lock with re-validation. + auto& new_bucket = build_result.value(); + // Determine which old buckets have ALL their live keys migrated. + // A key is "migrated" if it's in the first_group AND still maps to an + // old bucket being compacted. Old buckets with no remaining live keys + // (all migrated or removed) can be deleted. + std::set old_bucket_ids_set(bucket_ids.begin(), + bucket_ids.end()); + // Use the coldest last_access_ns among old buckets for the new bucket. + int64_t new_last_access_ns = std::numeric_limits::max(); + for (auto& [bid, pr] : old_buckets) { + new_last_access_ns = std::min(new_last_access_ns, pr.second); + } + if (new_last_access_ns == std::numeric_limits::max()) { + new_last_access_ns = 0; + } + + std::vector buckets_to_delete; + { + SharedMutexLocker lock(&mutex_); + for (const auto& [bid, pr] : old_buckets) { + auto current_it = buckets_.find(bid); + auto generation_it = old_bucket_generations.find(bid); + if (current_it == buckets_.end() || + generation_it == old_bucket_generations.end() || + current_it->second->generation_ != generation_it->second) { + // A delete changed the source bucket after the snapshot. The + // data file was built from a stale view, so do not publish it + // or replace any object mappings with it. + lock.unlock(); + CleanupOrphanedBucket(new_bucket_id); + reset_compacting(); + LOG(INFO) << "CompactBuckets discarded stale snapshot for " + << "bucket_id=" << bid; + return true; + } + } + // Re-validate and remap each key in the new bucket. + for (size_t i = 0; i < new_bucket->keys.size(); ++i) { + const auto& key = new_bucket->keys[i]; + auto map_it = object_bucket_map_.find(key); + if (map_it != object_bucket_map_.end() && + old_bucket_ids_set.count(map_it->second.bucket_id)) { + // Still live at an old bucket -> remap to new bucket. + map_it->second = new_metas[i]; + } + // else: key was removed or remapped -> skip. + } + + // Insert new bucket. + total_size_ += new_bucket->data_size + new_bucket->meta_size; + new_bucket->last_access_ns_.store( + new_last_access_ns, std::memory_order_relaxed); + buckets_.emplace(new_bucket_id, new_bucket); + lru_index_.emplace(new_last_access_ns, new_bucket_id); + + // Determine which old buckets can be deleted: those whose live keys + // are all now absent from object_bucket_map_ or remapped to the new + // bucket (i.e., no key still points at the old bucket). + for (auto& [bid, pr] : old_buckets) { + auto& old_bucket = pr.first; + bool has_remaining_live = false; + for (const auto& key : old_bucket->keys) { + auto map_it = object_bucket_map_.find(key); + if (map_it != object_bucket_map_.end() && + map_it->second.bucket_id == bid) { + // This key is still live at the old bucket — it was not + // in the first group (deferred to next round). + has_remaining_live = true; + break; + } + } + if (!has_remaining_live) { + buckets_to_delete.push_back(bid); + } else { + // Reset compacting_ so this bucket can be compacted later. + old_bucket->compacting_.store(false, + std::memory_order_relaxed); + } + } + + // Remove old buckets that are fully migrated. + for (int64_t bid : buckets_to_delete) { + auto it = buckets_.find(bid); + if (it != buckets_.end()) { + total_size_ -= + it->second->data_size + it->second->meta_size; + buckets_.erase(it); + int64_t ts = old_buckets[bid].second; + lru_index_.erase({ts, bid}); + } + } + } + + // Step 6: wait for inflight reads + delete old bucket files. + for (int64_t bid : buckets_to_delete) { + WaitForInflightReads(old_buckets[bid].first); + DeleteBucketFiles(bid); + } + + return true; +} + +void BucketStorageBackend::WaitForInflightReads( + std::shared_ptr bucket) { + constexpr int kMaxSpinIterations = 1000; + constexpr auto kMaxWaitTime = std::chrono::seconds(10); + int spin_count = 0; + auto wait_start = std::chrono::steady_clock::now(); + while (bucket->inflight_reads_.load(std::memory_order_acquire) > 0) { + if (++spin_count > kMaxSpinIterations) { + std::this_thread::yield(); + spin_count = 0; + if (std::chrono::steady_clock::now() - wait_start > + kMaxWaitTime) { + LOG(ERROR) << "CompactBucket: timed out waiting for " + "in-flight reads, inflight_reads=" + << bucket->inflight_reads_.load( + std::memory_order_relaxed); + break; + } + } else { + PAUSE(); + } + } +} + +void BucketStorageBackend::DeleteBucketFiles(int64_t bucket_id) { + namespace fs = std::filesystem; + std::error_code ec; + auto data_path = GetBucketDataPath(bucket_id); + if (data_path) { + { + MutexLocker cache_locker(&file_cache_mutex_); + file_cache_.erase(data_path.value()); + } + fs::remove(data_path.value(), ec); + if (ec && ec != std::errc::no_such_file_or_directory) { + LOG(ERROR) << "CompactBucket: failed to remove data file: " + << data_path.value() << ", error: " << ec.message(); + } + } + auto meta_path = GetBucketMetadataPath(bucket_id); + if (meta_path) { + ec.clear(); + fs::remove(meta_path.value(), ec); + if (ec && ec != std::errc::no_such_file_or_directory) { + LOG(ERROR) << "CompactBucket: failed to remove meta file: " + << meta_path.value() << ", error: " << ec.message(); + } + } +} + +// GCThreadFunc: background tombstone compaction loop. +void BucketStorageBackend::GCThreadFunc() { + LOG(INFO) << "[GC] background compaction thread started"; + while (gc_running_.load(std::memory_order_acquire)) { + // Sleep for gc_interval_ms or until woken for shutdown. + { + std::unique_lock lock(gc_mutex_); + gc_cv_.wait_for( + lock, + std::chrono::milliseconds( + bucket_backend_config_.gc_interval_ms), + [this]() { + return !gc_running_.load(std::memory_order_relaxed); + }); + } + if (!gc_running_.load(std::memory_order_acquire)) break; + + if (!bucket_backend_config_.gc_enable) continue; + + // Check space pressure under shared lock (total_size_ is + // GUARDED_BY(mutex_)). + bool space_pressure = false; + { + SharedMutexLocker lock(&mutex_, shared_lock); + if (bucket_backend_config_.max_total_size > 0) { + double used_ratio = + static_cast(total_size_) / + static_cast( + bucket_backend_config_.max_total_size); + space_pressure = used_ratio >= + bucket_backend_config_ + .gc_high_watermark_ratio; + } + } + + // Collect GC candidate buckets (up to gc_max_buckets_per_round). + std::vector candidates; + { + SharedMutexLocker lock(&mutex_); + int64_t count = 0; + for (auto it = buckets_.begin(); + it != buckets_.end() && + count < bucket_backend_config_.gc_max_buckets_per_round; + ++it) { + int64_t deleted = + it->second->deleted_bytes_.load( + std::memory_order_relaxed); + if (deleted <= 0) continue; + if (it->second->compacting_.load( + std::memory_order_relaxed)) + continue; + int64_t data_size = it->second->data_size; + double ratio = + (data_size > 0) + ? static_cast(deleted) / + static_cast(data_size) + : 0.0; + if (!space_pressure && + ratio < bucket_backend_config_.gc_deleted_ratio) { + continue; + } + candidates.push_back(it->first); + ++count; + } + } + + if (!candidates.empty()) { + if (bucket_backend_config_.gc_merge_enable && + candidates.size() > 1) { + // Cross-bucket merge: collect live keys from multiple + // tombstone buckets into one new bucket. + if (CompactBuckets(candidates, space_pressure)) { + LOG(INFO) << "[GC] merged " << candidates.size() + << " bucket(s)"; + } else { + LOG(WARNING) << "[GC] CompactBuckets failed for " + << candidates.size() + << " bucket(s), will retry next round"; + } + } else { + // Single-bucket compaction (one at a time). + int64_t compacted = 0; + for (int64_t bid : candidates) { + if (CompactBuckets({bid}, space_pressure)) { + ++compacted; + } else { + LOG(WARNING) + << "[GC] CompactBuckets failed for bucket " + << bid << ", will retry next round"; + break; + } + } + if (compacted > 0) { + LOG(INFO) << "[GC] compacted " << compacted + << " bucket(s)"; + } + } + } + } + LOG(INFO) << "[GC] background compaction thread stopped"; +} + tl::expected BucketStorageBackend::StoreBucketMetadata( int64_t id, std::shared_ptr metadata) { auto meta_path_res = GetBucketMetadataPath(id); diff --git a/mooncake-store/src/types.cpp b/mooncake-store/src/types.cpp index 7535abccbf..904836c552 100644 --- a/mooncake-store/src/types.cpp +++ b/mooncake-store/src/types.cpp @@ -50,6 +50,7 @@ const std::string& toString(ErrorCode errorCode) noexcept { {ErrorCode::UNAVAILABLE_IN_CURRENT_STATUS, "UNAVAILABLE_IN_CURRENT_STATUS"}, {ErrorCode::UNAVAILABLE_IN_CURRENT_MODE, "UNAVAILABLE_IN_CURRENT_MODE"}, + {ErrorCode::SLOT_NOT_OWNED, "SLOT_NOT_OWNED"}, {ErrorCode::FILE_NOT_FOUND, "FILE_NOT_FOUND"}, {ErrorCode::FILE_OPEN_FAIL, "FILE_OPEN_FAIL"}, {ErrorCode::FILE_READ_FAIL, "FILE_READ_FAIL"}, diff --git a/mooncake-store/src/vchunk_allocation_strategy.cpp b/mooncake-store/src/vchunk_allocation_strategy.cpp new file mode 100644 index 0000000000..de1327eabd --- /dev/null +++ b/mooncake-store/src/vchunk_allocation_strategy.cpp @@ -0,0 +1,144 @@ +#include "vchunk_allocation_strategy.h" + +#include +#include +#include + +#include "allocation_strategy.h" +#include "random.h" + +namespace mooncake { +namespace { + +struct Candidate { + std::string name; + const std::vector>* allocators; + uint64_t remaining_slices; + uint64_t allocated_slices{0}; +}; + +uint64_t AvailableBytes(const BufferAllocatorBase& allocator) { + const auto capacity = allocator.capacity(); + const auto used = allocator.size(); + if (capacity == kAllocatorUnknownFreeSpace) { + return allocator.getLargestFreeRegion(); + } + return capacity > used ? capacity - used : 0; +} + +std::unique_ptr AllocateFromCandidate(Candidate& candidate, + size_t size) { + for (const auto& allocator : *candidate.allocators) { + if (allocator && allocator->getLargestFreeRegion() >= size) { + if (auto buffer = allocator->allocate(size)) { + ++candidate.allocated_slices; + return buffer; + } + } + } + return nullptr; +} + +} // namespace + +tl::expected AllocateVChunk( + const AllocatorManager& allocator_manager, uint64_t total_size, + VCSliceSizeLevel slice_size_level, + const std::set& excluded_segments) { + const uint64_t slice_size = SliceSizeLevelToBytes(slice_size_level); + if (total_size == 0 || slice_size == 0 || + total_size > std::numeric_limits::max() - (slice_size - 1)) { + return tl::make_unexpected(ErrorCode::INVALID_PARAMS); + } + const uint64_t slice_count_u64 = + (total_size + slice_size - 1) / slice_size; + if (slice_count_u64 > std::numeric_limits::max()) { + return tl::make_unexpected(ErrorCode::INVALID_PARAMS); + } + + std::vector candidates; + for (const auto& name : allocator_manager.getNames()) { + if (excluded_segments.contains(name)) { + continue; + } + const auto* allocators = allocator_manager.getAllocators(name); + if (allocators == nullptr || allocators->empty()) { + continue; + } + uint64_t available = 0; + for (const auto& allocator : *allocators) { + if (!allocator) { + continue; + } + const auto bytes = AvailableBytes(*allocator); + if (available > std::numeric_limits::max() - bytes) { + available = std::numeric_limits::max(); + break; + } + available += bytes; + } + const uint64_t weight = available / slice_size; + if (weight > 0) { + candidates.push_back({name, allocators, weight, 0}); + } + } + if (candidates.empty()) { + return tl::make_unexpected(ErrorCode::NO_AVAILABLE_HANDLE); + } + + const auto slice_count = static_cast(slice_count_u64); + VChunkAllocationResult result; + result.row_size = std::min(slice_count, candidates.size()); + result.allocations.reserve(slice_count); + const size_t start_offset = randomIndex(candidates.size()); + + while (result.allocations.size() < slice_count) { + std::unordered_set used_in_row; + used_in_row.reserve(result.row_size); + const size_t row = result.allocations.size() / result.row_size; + const size_t row_width = std::min( + result.row_size, slice_count - result.allocations.size()); + for (size_t column = 0; column < row_width; ++column) { + bool allocated = false; + for (size_t attempt = 0; attempt < candidates.size(); ++attempt) { + const size_t index = + (start_offset + row + column + attempt) % candidates.size(); + auto& candidate = candidates[index]; + if (used_in_row.contains(candidate.name) || + candidate.allocated_slices >= candidate.remaining_slices) { + continue; + } + auto buffer = AllocateFromCandidate(candidate, slice_size); + if (!buffer) { + candidate.remaining_slices = candidate.allocated_slices; + continue; + } + + const auto slice_index = + static_cast(result.allocations.size()); + const uint64_t consumed = slice_index * slice_size; + const uint32_t logical_length = static_cast( + std::min(slice_size, total_size - consumed)); + VCSliceAllocation allocation; + allocation.slice_index = slice_index; + allocation.segment_name = candidate.name; + allocation.target_offset = + reinterpret_cast(buffer->data()); + allocation.logical_length = logical_length; + allocation.allocated_length = + static_cast(buffer->size()); + allocation.buffer = std::move(buffer); + result.allocations.push_back(std::move(allocation)); + used_in_row.insert(candidate.name); + allocated = true; + break; + } + if (!allocated) { + return tl::make_unexpected(ErrorCode::NO_AVAILABLE_HANDLE); + } + } + } + return result; +} + +} // namespace mooncake diff --git a/mooncake-store/src/vchunk_client.cpp b/mooncake-store/src/vchunk_client.cpp new file mode 100644 index 0000000000..aa66c31dab --- /dev/null +++ b/mooncake-store/src/vchunk_client.cpp @@ -0,0 +1,316 @@ +#include "vchunk_client.h" + +#include + +#include "master_service.h" + +namespace mooncake { +namespace { + +size_t SafeKeyId(const TenantId& tenant_id, const std::string& key) { + return std::hash{}(tenant_id.MakeScopedKey(key)); +} + +bool CountsForCircuitBreaker(ErrorCode error) { + return error == ErrorCode::NO_AVAILABLE_HANDLE || + error == ErrorCode::TRANSFER_FAIL || error == ErrorCode::RPC_TIMEOUT || + error == ErrorCode::ETCD_OPERATION_ERROR || + error == ErrorCode::RPC_FAIL; +} + +} // namespace + +VChunkClient::VChunkClient(bool enabled, MasterService& master, + VChunkDataPlane& data_plane, + VChunkLegacyPath& legacy, + std::chrono::milliseconds timeout, NowMs now_ms, + uint32_t max_retries, + uint32_t circuit_breaker_threshold, + std::shared_ptr metrics) + : enabled_(enabled), + owned_control_plane_(std::make_unique(master)), + control_plane_(owned_control_plane_.get()), + data_plane_(data_plane), + legacy_(legacy), + timeout_(timeout), + now_ms_(std::move(now_ms)), + max_retries_(max_retries), + circuit_breaker_threshold_(circuit_breaker_threshold), + metrics_(metrics ? std::move(metrics) + : std::make_shared()) {} + +VChunkClient::VChunkClient(bool enabled, VChunkControlPlane& control_plane, + VChunkDataPlane& data_plane, + VChunkLegacyPath& legacy, + std::chrono::milliseconds timeout, NowMs now_ms, + uint32_t max_retries, + uint32_t circuit_breaker_threshold, + std::shared_ptr metrics) + : enabled_(enabled), + control_plane_(&control_plane), + data_plane_(data_plane), + legacy_(legacy), + timeout_(timeout), + now_ms_(std::move(now_ms)), + max_retries_(max_retries), + circuit_breaker_threshold_(circuit_breaker_threshold), + metrics_(metrics ? std::move(metrics) + : std::make_shared()) {} + +ErrorCode VChunkClient::Put(const TenantId& tenant_id, const std::string& key, + const void* source, size_t length) { + const auto started = Clock::now(); + const auto deadline = started + timeout_; + if (!enabled_) { + return legacy_.Put(tenant_id, key, source, length); + } + if (circuit_breaker_threshold_ > 0 && + consecutive_put_failures_.load() >= circuit_breaker_threshold_) { + metrics_->Observe(VChunkOperation::PUT, false, 0); + return ErrorCode::NO_AVAILABLE_HANDLE; + } + if (!source || length == 0 || timeout_.count() <= 0 || !now_ms_) { + metrics_->Observe(VChunkOperation::PUT, false, 0); + return ErrorCode::INVALID_PARAMS; + } + auto created = control_plane_->PutStart(tenant_id, key, length, now_ms_()); + if (!created) { + if (CountsForCircuitBreaker(created.error())) { + consecutive_put_failures_.fetch_add(1); + } + metrics_->Observe( + VChunkOperation::PUT, false, + std::chrono::duration_cast(Clock::now() - + started) + .count()); + return created.error(); + } + if (Clock::now() >= deadline) { + (void)control_plane_->PutRevoke(tenant_id, key, created->vchunk_id); + metrics_->AddRollback(); + metrics_->AddTimeout(); + metrics_->Observe( + VChunkOperation::PUT, false, + std::chrono::duration_cast(Clock::now() - + started) + .count()); + return ErrorCode::RPC_TIMEOUT; + } + metrics_->AddSlices(created->slice_count); + metrics_->ObserveLayout(*created); + ErrorCode transfer = ErrorCode::TRANSFER_FAIL; + for (uint32_t attempt = 0; attempt <= max_retries_; ++attempt) { + if (Clock::now() >= deadline) { + transfer = ErrorCode::RPC_TIMEOUT; + break; + } + transfer = data_plane_.Write(*created, source, length, deadline); + if (transfer == ErrorCode::OK || + (transfer != ErrorCode::TRANSFER_FAIL && + transfer != ErrorCode::RPC_TIMEOUT)) { + break; + } + if (attempt < max_retries_) { + metrics_->AddRetry(); + } + } + if (transfer != ErrorCode::OK) { + if (!created->slices.empty()) { + const auto& slice = created->slices.front(); + LOG(ERROR) << "vchunk write failed tenant=" << tenant_id.value() + << " key_id=" << SafeKeyId(tenant_id, key) + << " vchunk_id=" << created->vchunk_id + << " status=" << static_cast(created->status) + << " slice_index=" << slice.slice_index + << " segment=" << slice.target_segment_name + << " slice_count=" << created->slice_count + << " error=" << static_cast(transfer); + } + const auto revoke = + control_plane_->PutRevoke(tenant_id, key, created->vchunk_id); + metrics_->AddRollback(); + if (transfer == ErrorCode::RPC_TIMEOUT) { + metrics_->AddTimeout(); + } else { + metrics_->AddTransferFailure(); + } + consecutive_put_failures_.fetch_add(1); + const auto result = revoke == ErrorCode::OK ? transfer : revoke; + metrics_->Observe( + VChunkOperation::PUT, false, + std::chrono::duration_cast(Clock::now() - + started) + .count()); + return result; + } + if (Clock::now() >= deadline) { + (void)control_plane_->PutRevoke(tenant_id, key, created->vchunk_id); + metrics_->AddRollback(); + metrics_->AddTimeout(); + metrics_->Observe( + VChunkOperation::PUT, false, + std::chrono::duration_cast(Clock::now() - + started) + .count()); + return ErrorCode::RPC_TIMEOUT; + } + auto end = + control_plane_->PutEnd(tenant_id, key, created->vchunk_id, now_ms_()); + // PutEnd may have committed durably while its RPC response was lost. Read + // back before revoking so an ambiguous timeout cannot leave a successful + // object reported as failed (or turn it into an orphan). + if (end != ErrorCode::OK) { + auto committed = control_plane_->Get(tenant_id, key); + if (committed && + committed->record.vchunk_id == created->vchunk_id && + committed->record.status == VChunkStatus::ACTIVE) { + end = ErrorCode::OK; + } + } + if (end != ErrorCode::OK) { + control_plane_->PutRevoke(tenant_id, key, created->vchunk_id); + } + if (end == ErrorCode::OK) { + consecutive_put_failures_.store(0); + } else { + consecutive_put_failures_.fetch_add(1); + } + metrics_->Observe( + VChunkOperation::PUT, end == ErrorCode::OK, + std::chrono::duration_cast(Clock::now() - + started) + .count()); + return end; +} + +ErrorCode VChunkClient::Get(const TenantId& tenant_id, const std::string& key, + void* destination, size_t length) { + const auto started = Clock::now(); + const auto deadline = started + timeout_; + if (!enabled_) { + return legacy_.Get(tenant_id, key, destination, length); + } + if (!destination || timeout_.count() <= 0) { + metrics_->Observe(VChunkOperation::GET, false, 0); + return ErrorCode::INVALID_PARAMS; + } + auto read = control_plane_->Get(tenant_id, key); + if (!read) { + metrics_->Observe(VChunkOperation::GET, false, 0); + return read.error(); + } + if (read->record.total_size != length) { + metrics_->Observe(VChunkOperation::GET, false, 0); + return ErrorCode::INVALID_PARAMS; + } + if (Clock::now() >= deadline) { + metrics_->AddTimeout(); + metrics_->Observe( + VChunkOperation::GET, false, + std::chrono::duration_cast(Clock::now() - + started) + .count()); + return ErrorCode::RPC_TIMEOUT; + } + ErrorCode result = ErrorCode::TRANSFER_FAIL; + for (uint32_t attempt = 0; attempt <= max_retries_; ++attempt) { + if (Clock::now() >= deadline) { + result = ErrorCode::RPC_TIMEOUT; + break; + } + result = data_plane_.Read(read->record, destination, length, deadline); + if (result == ErrorCode::OK || + (result != ErrorCode::TRANSFER_FAIL && + result != ErrorCode::RPC_TIMEOUT)) { + break; + } + if (attempt < max_retries_) { + metrics_->AddRetry(); + } + } + if (result == ErrorCode::RPC_TIMEOUT) { + metrics_->AddTimeout(); + } else if (result == ErrorCode::TRANSFER_FAIL) { + metrics_->AddTransferFailure(); + } + if (result != ErrorCode::OK) { + if (!read->record.slices.empty()) { + const auto& slice = read->record.slices.front(); + LOG(ERROR) << "vchunk read failed tenant=" << tenant_id.value() + << " key_id=" << SafeKeyId(tenant_id, key) + << " vchunk_id=" << read->record.vchunk_id + << " status=" << static_cast(read->record.status) + << " slice_index=" << slice.slice_index + << " segment=" << slice.target_segment_name + << " slice_count=" << read->record.slice_count + << " error=" << static_cast(result); + } + } + metrics_->Observe( + VChunkOperation::GET, result == ErrorCode::OK, + std::chrono::duration_cast(Clock::now() - + started) + .count()); + return result; +} + +ErrorCode VChunkClient::Remove(const TenantId& tenant_id, + const std::string& key) { + const auto started = Clock::now(); + if (!enabled_) { + return legacy_.Remove(tenant_id, key); + } + if (!now_ms_) { + metrics_->Observe(VChunkOperation::REMOVE, false, 0); + return ErrorCode::INVALID_PARAMS; + } + auto result = control_plane_->Remove(tenant_id, key, now_ms_()); + if (result == ErrorCode::RPC_TIMEOUT || result == ErrorCode::RPC_FAIL) { + metrics_->AddRetry(); + result = control_plane_->Remove(tenant_id, key, now_ms_()); + } + metrics_->Observe( + VChunkOperation::REMOVE, result == ErrorCode::OK, + std::chrono::duration_cast(Clock::now() - + started) + .count()); + return result; +} + +std::vector VChunkClient::BatchPut( + const TenantId& tenant_id, const std::vector& requests) { + std::vector results; + results.reserve(requests.size()); + for (const auto& request : requests) { + results.push_back( + Put(tenant_id, request.key, request.source, request.length)); + } + return results; +} + +std::vector VChunkClient::BatchGet( + const TenantId& tenant_id, const std::vector& requests) { + std::vector results; + results.reserve(requests.size()); + for (const auto& request : requests) { + results.push_back(Get(tenant_id, request.key, request.destination, + request.length)); + } + return results; +} + +std::vector VChunkClient::BatchRemove( + const TenantId& tenant_id, const std::vector& keys) { + std::vector results; + results.reserve(keys.size()); + for (const auto& key : keys) { + results.push_back(Remove(tenant_id, key)); + } + return results; +} + +VChunkMetricsSnapshot VChunkClient::MetricsSnapshot() const { + return metrics_->Snapshot(); +} + +} // namespace mooncake diff --git a/mooncake-store/src/vchunk_config.cpp b/mooncake-store/src/vchunk_config.cpp new file mode 100644 index 0000000000..93df6ccc28 --- /dev/null +++ b/mooncake-store/src/vchunk_config.cpp @@ -0,0 +1,29 @@ +#include "vchunk_config.h" + +namespace mooncake { + +VCSliceSizeLevel SelectVChunkSliceSize(uint64_t value_size, + bool is_ssd_segment) { + if (is_ssd_segment || value_size < 64U * 1024U) { + return VCSliceSizeLevel::k4K; + } + if (value_size < 256U * 1024U) { + return VCSliceSizeLevel::k64K; + } + if (value_size < 1024U * 1024U) { + return VCSliceSizeLevel::k256K; + } + return VCSliceSizeLevel::k1M; +} + +ErrorCode VChunkConfig::Validate() const { + if (creating_timeout_ms == 0 || releasing_timeout_ms == 0 || + max_slice_count == 0 || max_metadata_bytes == 0 || + max_creating_objects == 0 || reaper_interval_ms == 0 || + reaper_max_scan == 0) { + return ErrorCode::INVALID_PARAMS; + } + return ErrorCode::OK; +} + +} // namespace mooncake diff --git a/mooncake-store/src/vchunk_control_plane.cpp b/mooncake-store/src/vchunk_control_plane.cpp new file mode 100644 index 0000000000..fa44c85e7c --- /dev/null +++ b/mooncake-store/src/vchunk_control_plane.cpp @@ -0,0 +1,107 @@ +#include "vchunk_control_plane.h" + +#include "master_client.h" +#include "master_service.h" + +namespace mooncake { + +namespace { + +class RpcReadLeaseGuard { + public: + RpcReadLeaseGuard(MasterClient& master, std::string tenant_id, + std::string key, std::string lease_id) + : master_(master), + tenant_id_(std::move(tenant_id)), + key_(std::move(key)), + lease_id_(std::move(lease_id)) {} + + ~RpcReadLeaseGuard() { + (void)master_.ReleaseVChunkReadLease(tenant_id_, key_, lease_id_); + } + + private: + MasterClient& master_; + std::string tenant_id_; + std::string key_; + std::string lease_id_; +}; + +} // namespace + +tl::expected +LocalVChunkControlPlane::PutStart(const TenantId& tenant_id, + const std::string& key, uint64_t total_size, + int64_t now_ms) { + return master_.VChunkPutStart(tenant_id, key, total_size, false, now_ms); +} + +ErrorCode LocalVChunkControlPlane::PutEnd(const TenantId& tenant_id, + const std::string& key, + const std::string& vchunk_id, + int64_t now_ms) { + return master_.VChunkPutEnd(tenant_id, key, vchunk_id, now_ms); +} + +ErrorCode LocalVChunkControlPlane::PutRevoke(const TenantId& tenant_id, + const std::string& key, + const std::string& vchunk_id) { + return master_.VChunkPutRevoke(tenant_id, key, vchunk_id); +} + +tl::expected LocalVChunkControlPlane::Get( + const TenantId& tenant_id, const std::string& key) { + auto handle = master_.AcquireVChunkRead(tenant_id, key); + if (!handle) return tl::unexpected(handle.error()); + auto lifetime = std::make_shared( + std::move(*handle)); + return VChunkControlPlaneRead{lifetime->record(), std::move(lifetime)}; +} + +ErrorCode LocalVChunkControlPlane::Remove(const TenantId& tenant_id, + const std::string& key, + int64_t now_ms) { + return master_.RemoveVChunk(tenant_id, key, now_ms); +} + +tl::expected +RpcVChunkControlPlane::PutStart(const TenantId& tenant_id, + const std::string& key, uint64_t total_size, + int64_t now_ms) { + return master_.VChunkPutStart(tenant_id.value(), key, total_size, now_ms); +} + +ErrorCode RpcVChunkControlPlane::PutEnd(const TenantId& tenant_id, + const std::string& key, + const std::string& vchunk_id, + int64_t now_ms) { + auto result = + master_.VChunkPutEnd(tenant_id.value(), key, vchunk_id, now_ms); + return result ? ErrorCode::OK : result.error(); +} + +ErrorCode RpcVChunkControlPlane::PutRevoke(const TenantId& tenant_id, + const std::string& key, + const std::string& vchunk_id) { + auto result = master_.VChunkPutRevoke(tenant_id.value(), key, vchunk_id); + return result ? ErrorCode::OK : result.error(); +} + +tl::expected RpcVChunkControlPlane::Get( + const TenantId& tenant_id, const std::string& key) { + auto record = master_.GetVChunk(tenant_id.value(), key); + if (!record) return tl::unexpected(record.error()); + auto lifetime = std::make_shared( + master_, tenant_id.value(), key, record->lease_id); + return VChunkControlPlaneRead{std::move(record->record), + std::move(lifetime)}; +} + +ErrorCode RpcVChunkControlPlane::Remove(const TenantId& tenant_id, + const std::string& key, + int64_t now_ms) { + auto result = master_.RemoveVChunk(tenant_id.value(), key, now_ms); + return result ? ErrorCode::OK : result.error(); +} + +} // namespace mooncake diff --git a/mooncake-store/src/vchunk_master_manager.cpp b/mooncake-store/src/vchunk_master_manager.cpp new file mode 100644 index 0000000000..995e83b562 --- /dev/null +++ b/mooncake-store/src/vchunk_master_manager.cpp @@ -0,0 +1,371 @@ +#include "vchunk_master_manager.h" + +#include +#include +#include +#include + +#include "types.h" + +namespace mooncake { + +VChunkMasterManager::VChunkMasterManager( + VChunkConfig config, std::shared_ptr metadata_store, + std::shared_ptr metrics) + : config_(std::move(config)), + metadata_store_(metadata_store ? std::move(metadata_store) + : std::make_shared< + InMemoryVChunkMetadataStore>()), + metrics_(metrics ? std::move(metrics) + : std::make_shared()) {} + +std::string VChunkMasterManager::ScopedKey(const TenantId& tenant_id, + const std::string& key) { + return tenant_id.MakeScopedKey(key); +} + +tl::expected VChunkMasterManager::PutStart( + const AllocatorManager& allocator_manager, const TenantId& tenant_id, + const std::string& key, uint64_t total_size, bool is_ssd_segment, + int64_t now_ms, const std::set& excluded_segments) { + if (!config_.enabled || config_.Validate() != ErrorCode::OK || + !tenant_id.IsValid() || key.empty() || total_size == 0 || now_ms < 0) { + return tl::make_unexpected(ErrorCode::INVALID_PARAMS); + } + // The current version owns buffers from SegmentManager and supports + // memory segments only. SSD/NoF routing is introduced in a later stage. + if (is_ssd_segment) { + return tl::make_unexpected(ErrorCode::INVALID_PARAMS); + } + const auto scoped_key = ScopedKey(tenant_id, key); + { + std::lock_guard guard(mutex_); + if (entries_.contains(scoped_key) || pending_puts_.contains(scoped_key)) { + return tl::make_unexpected(ErrorCode::OBJECT_ALREADY_EXISTS); + } + size_t creating = 0; + for (const auto& [_, entry] : entries_) { + creating += entry->record.status == VChunkStatus::CREATING; + } + if (creating + pending_puts_.size() >= config_.max_creating_objects) { + return tl::make_unexpected(ErrorCode::NO_AVAILABLE_HANDLE); + } + pending_puts_.insert(scoped_key); + } + + const auto slice_size_level = + SelectVChunkSliceSize(total_size, is_ssd_segment); + const uint64_t slice_size = SliceSizeLevelToBytes(slice_size_level); + if (total_size > std::numeric_limits::max() - (slice_size - 1) || + (total_size + slice_size - 1) / slice_size > + config_.max_slice_count) { + ReleasePendingPut(scoped_key); + return tl::make_unexpected(ErrorCode::INVALID_PARAMS); + } + + auto allocation = AllocateVChunk(allocator_manager, total_size, + slice_size_level, excluded_segments); + if (!allocation) { + ReleasePendingPut(scoped_key); + metrics_->AddAllocationFailure(); + return tl::make_unexpected(allocation.error()); + } + + auto entry = std::make_shared(); + auto& record = entry->record; + record.vchunk_id = UuidToString(generate_uuid()); + record.tenant_id = tenant_id.value(); + record.key = key; + record.total_size = total_size; + record.slice_count = + static_cast(allocation->allocations.size()); + record.slice_size_level = slice_size_level; + record.row_size = static_cast(allocation->row_size); + record.status = VChunkStatus::CREATING; + record.created_at_ms = now_ms; + record.last_updated_at_ms = now_ms; + record.slices.reserve(record.slice_count); + entry->buffers.reserve(record.slice_count); + for (auto& allocated : allocation->allocations) { + record.slices.push_back(VCSliceDescriptor{ + allocated.slice_index, allocated.segment_name, + allocated.target_offset, allocated.logical_length, + allocated.allocated_length, VCSliceStatus::PENDING, 0}); + entry->buffers.push_back(std::move(allocated.buffer)); + } + const auto serialized = SerializeVChunkMetadata(record, config_); + if (!serialized) { + ReleasePendingPut(scoped_key); + return tl::make_unexpected(serialized.error()); + } + if (const auto error = metadata_store_->Put(record); + error != ErrorCode::OK) { + ReleasePendingPut(scoped_key); + return tl::make_unexpected(error); + } + + std::lock_guard guard(mutex_); + pending_puts_.erase(scoped_key); + const auto snapshot = record; + entries_.emplace(scoped_key, std::move(entry)); + metrics_->AddSlices(snapshot.slice_count); + metrics_->ObserveLayout(snapshot); + metrics_->AddMetadataBytes(serialized->size()); + RefreshStateMetricsLocked(); + return snapshot; +} + +ErrorCode VChunkMasterManager::PutEnd(const TenantId& tenant_id, + const std::string& key, + const std::string& vchunk_id, + int64_t now_ms) { + std::lock_guard guard(mutex_); + const auto it = entries_.find(ScopedKey(tenant_id, key)); + if (it == entries_.end()) { + return ErrorCode::OBJECT_NOT_FOUND; + } + auto& record = it->second->record; + if (record.vchunk_id != vchunk_id) { + return ErrorCode::INVALID_VERSION; + } + if (record.status == VChunkStatus::ACTIVE) { + return ErrorCode::OK; + } + if (ValidateVChunkTransition(record.status, VChunkStatus::ACTIVE) != + ErrorCode::OK || + now_ms < record.last_updated_at_ms) { + return ErrorCode::INVALID_PARAMS; + } + auto durable = record; + for (auto& slice : durable.slices) { + slice.status = VCSliceStatus::COMPLETED; + } + durable.status = VChunkStatus::ACTIVE; + durable.last_updated_at_ms = now_ms; + if (const auto error = metadata_store_->Put(durable); + error != ErrorCode::OK) { + return error; + } + record = std::move(durable); + RefreshStateMetricsLocked(); + return ErrorCode::OK; +} + +ErrorCode VChunkMasterManager::PutRevoke(const TenantId& tenant_id, + const std::string& key, + const std::string& vchunk_id) { + std::lock_guard guard(mutex_); + const auto it = entries_.find(ScopedKey(tenant_id, key)); + if (it == entries_.end()) { + return ErrorCode::OK; + } + if (it->second->record.vchunk_id != vchunk_id) { + return ErrorCode::INVALID_VERSION; + } + if (it->second->record.status != VChunkStatus::CREATING && + it->second->record.status != VChunkStatus::FAILED) { + return ErrorCode::INVALID_PARAMS; + } + if (const auto error = metadata_store_->Remove(it->second->record); + error != ErrorCode::OK) { + return error; + } + entries_.erase(it); + metrics_->AddRollback(); + RefreshStateMetricsLocked(); + return ErrorCode::OK; +} + +tl::expected VChunkMasterManager::Get( + const TenantId& tenant_id, const std::string& key) const { + auto handle = AcquireRead(tenant_id, key); + if (!handle) { + return tl::make_unexpected(handle.error()); + } + return handle->record(); +} + +tl::expected +VChunkMasterManager::AcquireRead(const TenantId& tenant_id, + const std::string& key) const { + std::lock_guard guard(mutex_); + const auto it = entries_.find(ScopedKey(tenant_id, key)); + if (it == entries_.end()) { + return tl::make_unexpected(ErrorCode::OBJECT_NOT_FOUND); + } + if (it->second->record.status != VChunkStatus::ACTIVE) { + return tl::make_unexpected(ErrorCode::REPLICA_IS_NOT_READY); + } + ReadHandle handle; + handle.record_ = it->second->record; + handle.lifetime_ = it->second; + return handle; +} + +ErrorCode VChunkMasterManager::Remove(const TenantId& tenant_id, + const std::string& key, + int64_t now_ms) { + std::lock_guard guard(mutex_); + const auto scoped_key = ScopedKey(tenant_id, key); + const auto it = entries_.find(scoped_key); + if (it == entries_.end()) { + return ErrorCode::OK; + } + auto& record = it->second->record; + if ((record.status != VChunkStatus::ACTIVE && + record.status != VChunkStatus::RELEASING) || + now_ms < record.last_updated_at_ms) { + return ErrorCode::INVALID_PARAMS; + } + if (record.status == VChunkStatus::ACTIVE) { + auto releasing = record; + releasing.status = VChunkStatus::RELEASING; + releasing.last_updated_at_ms = now_ms; + if (const auto error = metadata_store_->Put(releasing); + error != ErrorCode::OK) { + return error; + } + record = std::move(releasing); + } + if (const auto error = metadata_store_->Remove(record); + error != ErrorCode::OK) { + return error; + } + entries_.erase(it); + RefreshStateMetricsLocked(); + return ErrorCode::OK; +} + +ErrorCode VChunkMasterManager::Recover(int64_t now_ms, + OwnershipPredicate owns) { + if (now_ms < 0) { + return ErrorCode::INVALID_PARAMS; + } + auto records = metadata_store_->List(); + if (!records) { + return records.error(); + } + std::lock_guard guard(mutex_); + // Validate the complete snapshot before mutating the store so recovery is + // deterministic even when List() returns records in a different order. + for (const auto& record : *records) { + if (owns && !owns(record)) { + continue; + } + const auto validation = ValidateVChunkMetadata(record, config_); + if (validation != ErrorCode::OK) { + return validation; + } + // Allocator reservations are process-local. A persisted ACTIVE record + // must never be published until its exact ranges have been reserved + // again, otherwise new allocations can overlap it. + if (record.status == VChunkStatus::ACTIVE) { + return ErrorCode::REPLICA_IS_GONE; + } + } + for (const auto& record : *records) { + if (owns && !owns(record)) { + continue; + } + // CREATING records cannot be resumed safely either: their buffers were + // owned by the previous process. Treat all incomplete writes as stale. + if (record.status == VChunkStatus::CREATING || + record.status == VChunkStatus::RELEASING || + record.status == VChunkStatus::RELEASED || + record.status == VChunkStatus::FAILED) { + const auto error = metadata_store_->Remove(record); + if (error != ErrorCode::OK) { + return error; + } + continue; + } + } + RefreshStateMetricsLocked(); + return ErrorCode::OK; +} + +tl::expected VChunkMasterManager::ReapExpired( + int64_t now_ms, size_t max_scan, OwnershipPredicate owns) { + if (now_ms < 0 || max_scan == 0) { + return tl::make_unexpected(ErrorCode::INVALID_PARAMS); + } + std::lock_guard guard(mutex_); + size_t scanned = 0; + size_t removed = 0; + if (entries_.empty()) { + reaper_cursor_key_.clear(); + return removed; + } + auto it = reaper_cursor_key_.empty() ? entries_.begin() + : entries_.find(reaper_cursor_key_); + if (it == entries_.end()) it = entries_.begin(); + const size_t scan_limit = std::min(max_scan, entries_.size()); + while (!entries_.empty() && scanned < scan_limit) { + ++scanned; + auto next = std::next(it); + if (next == entries_.end()) next = entries_.begin(); + const auto& record = it->second->record; + if (owns && !owns(record)) { + it = next; + continue; + } + const bool time_is_valid = now_ms >= record.last_updated_at_ms; + const auto age = time_is_valid + ? static_cast(now_ms - + record.last_updated_at_ms) + : 0; + const bool expired = + time_is_valid && + ((record.status == VChunkStatus::CREATING && + age >= config_.creating_timeout_ms) || + (record.status == VChunkStatus::RELEASING && + age >= config_.releasing_timeout_ms)); + if (!expired) { + it = next; + continue; + } + if (const auto error = metadata_store_->Remove(record); + error != ErrorCode::OK) { + return tl::make_unexpected(error); + } + entries_.erase(it); + ++removed; + metrics_->AddRollback(); + if (entries_.empty()) break; + it = next; + } + reaper_cursor_key_ = entries_.empty() ? std::string() : it->first; + RefreshStateMetricsLocked(); + return removed; +} + +VChunkMetricsSnapshot VChunkMasterManager::MetricsSnapshot() const { + return metrics_->Snapshot(); +} + +void VChunkMasterManager::RefreshStateMetricsLocked() { + std::array counts{}; + uint64_t allocated_bytes = 0; + for (const auto& [_, entry] : entries_) { + ++counts[static_cast(entry->record.status)]; + for (const auto& buffer : entry->buffers) { + if (buffer) allocated_bytes += buffer->size(); + } + } + for (size_t i = 0; i < counts.size(); ++i) { + metrics_->SetStateCount(static_cast(i), counts[i]); + } + metrics_->SetAllocatedBytes(allocated_bytes); +} + +size_t VChunkMasterManager::SizeForTesting() const { + std::lock_guard guard(mutex_); + return entries_.size(); +} + +void VChunkMasterManager::ReleasePendingPut(const std::string& scoped_key) { + std::lock_guard guard(mutex_); + pending_puts_.erase(scoped_key); +} + +} // namespace mooncake diff --git a/mooncake-store/src/vchunk_metadata.cpp b/mooncake-store/src/vchunk_metadata.cpp new file mode 100644 index 0000000000..2928d129d7 --- /dev/null +++ b/mooncake-store/src/vchunk_metadata.cpp @@ -0,0 +1,191 @@ +#include "vchunk_metadata.h" + +#include +#include +#include + +namespace mooncake { +namespace { + +bool IsKnownSliceSize(VCSliceSizeLevel level) { + switch (level) { + case VCSliceSizeLevel::k4K: + case VCSliceSizeLevel::k64K: + case VCSliceSizeLevel::k256K: + case VCSliceSizeLevel::k1M: + return true; + } + return false; +} + +bool IsKnownSliceStatus(VCSliceStatus status) { + switch (status) { + case VCSliceStatus::PENDING: + case VCSliceStatus::COMPLETED: + case VCSliceStatus::FAILED: + return true; + } + return false; +} + +bool IsKnownVChunkStatus(VChunkStatus status) { + switch (status) { + case VChunkStatus::CREATING: + case VChunkStatus::ACTIVE: + case VChunkStatus::RELEASING: + case VChunkStatus::RELEASED: + case VChunkStatus::FAILED: + return true; + } + return false; +} + +} // namespace + +ErrorCode ValidateVChunkMetadata(const VChunkMetadataRecord& record, + const VChunkConfig& config) { + if (config.Validate() != ErrorCode::OK) { + return ErrorCode::INVALID_PARAMS; + } + if (record.schema_version != kVChunkMetadataSchemaVersion) { + return ErrorCode::INVALID_VERSION; + } + if (record.vchunk_id.empty() || record.tenant_id.empty() || + record.key.empty() || record.total_size == 0 || + !IsKnownSliceSize(record.slice_size_level) || + !IsKnownVChunkStatus(record.status)) { + return ErrorCode::INVALID_PARAMS; + } + if (record.slice_count == 0 || + record.slice_count > config.max_slice_count || + record.slices.size() != record.slice_count || record.row_size == 0 || + record.row_size > record.slice_count || record.created_at_ms < 0 || + record.last_updated_at_ms < record.created_at_ms) { + return ErrorCode::INVALID_PARAMS; + } + + const uint64_t slice_size = + SliceSizeLevelToBytes(record.slice_size_level); + uint64_t covered_bytes = 0; + std::unordered_set segments_in_row; + segments_in_row.reserve(record.row_size); + for (uint32_t i = 0; i < record.slice_count; ++i) { + const auto& slice = record.slices[i]; + if (slice.slice_index != i || slice.target_segment_name.empty() || + slice.logical_length == 0 || + slice.logical_length > slice.allocated_length || + slice.allocated_length < slice_size || + !IsKnownSliceStatus(slice.status) || + slice.retry_count > config.max_slice_retry) { + return ErrorCode::INVALID_PARAMS; + } + if (slice.target_offset > + std::numeric_limits::max() - slice.allocated_length) { + return ErrorCode::INVALID_PARAMS; + } + if (covered_bytes > + std::numeric_limits::max() - slice.logical_length) { + return ErrorCode::INVALID_PARAMS; + } + covered_bytes += slice.logical_length; + + if (i % record.row_size == 0) { + segments_in_row.clear(); + } + if (!segments_in_row.insert(slice.target_segment_name).second) { + return ErrorCode::INVALID_PARAMS; + } + } + + if (covered_bytes != record.total_size) { + return ErrorCode::INVALID_PARAMS; + } + return ErrorCode::OK; +} + +ErrorCode ValidateVChunkTransition(VChunkStatus from, VChunkStatus to) { + if (!IsKnownVChunkStatus(from) || !IsKnownVChunkStatus(to)) { + return ErrorCode::INVALID_PARAMS; + } + if (from == to) { + return ErrorCode::OK; + } + switch (from) { + case VChunkStatus::CREATING: + return (to == VChunkStatus::ACTIVE || to == VChunkStatus::FAILED) + ? ErrorCode::OK + : ErrorCode::INVALID_PARAMS; + case VChunkStatus::ACTIVE: + return (to == VChunkStatus::RELEASING || + to == VChunkStatus::FAILED) + ? ErrorCode::OK + : ErrorCode::INVALID_PARAMS; + case VChunkStatus::RELEASING: + return (to == VChunkStatus::RELEASED || + to == VChunkStatus::FAILED) + ? ErrorCode::OK + : ErrorCode::INVALID_PARAMS; + case VChunkStatus::FAILED: + return to == VChunkStatus::RELEASING ? ErrorCode::OK + : ErrorCode::INVALID_PARAMS; + case VChunkStatus::RELEASED: + return ErrorCode::INVALID_PARAMS; + } + return ErrorCode::INVALID_PARAMS; +} + +tl::expected, ErrorCode> SerializeVChunkMetadata( + const VChunkMetadataRecord& record, const VChunkConfig& config) { + const auto validation = ValidateVChunkMetadata(record, config); + if (validation != ErrorCode::OK) { + return tl::make_unexpected(validation); + } + auto bytes = struct_pack::serialize(record); + if (bytes.size() > config.max_metadata_bytes) { + return tl::make_unexpected(ErrorCode::BUFFER_OVERFLOW); + } + return bytes; +} + +tl::expected DeserializeVChunkMetadata( + const std::vector& bytes, const VChunkConfig& config) { + if (config.Validate() != ErrorCode::OK || bytes.empty()) { + return tl::make_unexpected(ErrorCode::INVALID_PARAMS); + } + if (bytes.size() > config.max_metadata_bytes) { + return tl::make_unexpected(ErrorCode::BUFFER_OVERFLOW); + } + + VChunkMetadataRecord record; + if (struct_pack::deserialize_to(record, bytes) != struct_pack::errc::ok) { + return tl::make_unexpected(ErrorCode::INVALID_PARAMS); + } + const auto validation = ValidateVChunkMetadata(record, config); + if (validation != ErrorCode::OK) { + return tl::make_unexpected(validation); + } + return record; +} + +VChunkMetadata::VChunkMetadata(VChunkMetadataRecord record) + : record_(std::move(record)) {} + +VChunkMetadataRecord VChunkMetadata::Snapshot() const { + SpinLocker guard(&lock_); + return record_; +} + +ErrorCode VChunkMetadata::TransitionTo(VChunkStatus next, + int64_t updated_at_ms) { + SpinLocker guard(&lock_); + const auto validation = ValidateVChunkTransition(record_.status, next); + if (validation != ErrorCode::OK || + updated_at_ms < record_.last_updated_at_ms) { + return ErrorCode::INVALID_PARAMS; + } + record_.status = next; + record_.last_updated_at_ms = updated_at_ms; + return ErrorCode::OK; +} + +} // namespace mooncake diff --git a/mooncake-store/src/vchunk_metadata_store.cpp b/mooncake-store/src/vchunk_metadata_store.cpp new file mode 100644 index 0000000000..594e60e65c --- /dev/null +++ b/mooncake-store/src/vchunk_metadata_store.cpp @@ -0,0 +1,200 @@ +#include "vchunk_metadata_store.h" + +#include + +#include +#include + +#if __has_include() +#include +#else +#include +#endif + +#include "etcd_helper.h" + +namespace mooncake { +namespace { + +std::string HexEncode(std::string_view value) { + std::ostringstream stream; + stream << std::hex << std::setfill('0'); + for (const unsigned char byte : value) { + stream << std::setw(2) << static_cast(byte); + } + return stream.str(); +} + +tl::expected, ErrorCode> HexDecode(std::string_view value) { + if (value.size() % 2 != 0) { + return tl::unexpected(ErrorCode::INVALID_VERSION); + } + auto digit = [](char c) -> int { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + return -1; + }; + std::vector bytes; + bytes.reserve(value.size() / 2); + for (size_t i = 0; i < value.size(); i += 2) { + const int high = digit(value[i]); + const int low = digit(value[i + 1]); + if (high < 0 || low < 0) { + return tl::unexpected(ErrorCode::INVALID_VERSION); + } + bytes.push_back(static_cast((high << 4) | low)); + } + return bytes; +} + +std::string MakeObjectIndexKey(std::string_view namespace_prefix, + const VChunkMetadataRecord& record) { + return std::string(namespace_prefix) + "/objects/" + + HexEncode(record.tenant_id) + "/" + HexEncode(record.key); +} + +std::string MakePersistentRecordKey(std::string_view namespace_prefix, + const VChunkMetadataRecord& record) { + return std::string(namespace_prefix) + "/records/" + + HexEncode(record.tenant_id) + "/" + record.vchunk_id; +} + +std::string BytesToString(const std::vector& bytes) { + return {bytes.data(), bytes.size()}; +} + +} // namespace + +std::string MakeVChunkMetadataStoreKey(const VChunkMetadataRecord& record) { + return std::string(kVChunkMetadataNamespace) + "/" + record.tenant_id + + "/" + record.vchunk_id; +} + +ErrorCode InMemoryVChunkMetadataStore::Put( + const VChunkMetadataRecord& record) { + std::lock_guard guard(mutex_); + records_[MakeVChunkMetadataStoreKey(record)] = record; + return ErrorCode::OK; +} + +ErrorCode InMemoryVChunkMetadataStore::Remove( + const VChunkMetadataRecord& record) { + std::lock_guard guard(mutex_); + records_.erase(MakeVChunkMetadataStoreKey(record)); + return ErrorCode::OK; +} + +tl::expected, ErrorCode> +InMemoryVChunkMetadataStore::List() { + std::lock_guard guard(mutex_); + std::vector result; + result.reserve(records_.size()); + for (const auto& [_, record] : records_) { + result.push_back(record); + } + return result; +} + +EtcdVChunkMetadataStore::EtcdVChunkMetadataStore(std::string endpoints, + VChunkConfig config, + std::string cluster_id) + : endpoints_(std::move(endpoints)), + config_(std::move(config)), + namespace_prefix_(std::string(kVChunkMetadataNamespace) + "/clusters/" + + HexEncode(cluster_id)) { +#ifdef STORE_USE_ETCD + connection_error_ = EtcdHelper::ConnectToEtcdStoreClient(endpoints_); +#else + connection_error_ = ErrorCode::ETCD_OPERATION_ERROR; +#endif +} + +ErrorCode EtcdVChunkMetadataStore::Put(const VChunkMetadataRecord& record) { + if (connection_error_ != ErrorCode::OK) return connection_error_; + auto encoded = SerializeVChunkMetadata(record, config_); + if (!encoded) return encoded.error(); + const auto metadata_key = MakePersistentRecordKey(namespace_prefix_, record); + const auto object_key = MakeObjectIndexKey(namespace_prefix_, record); + const auto value = HexEncode(BytesToString(*encoded)); + + if (record.status == VChunkStatus::CREATING) { + const auto error = EtcdHelper::TxnCompareAndPut( + {{object_key, EtcdHelper::TxnCompareKind::kKeyNotExists, {}}, + {metadata_key, EtcdHelper::TxnCompareKind::kKeyNotExists, {}}}, + {{object_key, record.vchunk_id}, {metadata_key, value}}); + return error == ErrorCode::ETCD_TRANSACTION_FAIL + ? ErrorCode::OBJECT_ALREADY_EXISTS + : error; + } + + std::string current; + EtcdRevisionId revision = 0; + auto error = EtcdHelper::Get(metadata_key.data(), metadata_key.size(), + current, revision); + if (error != ErrorCode::OK) return error; + return EtcdHelper::TxnCompareAndPut( + {{object_key, EtcdHelper::TxnCompareKind::kValueEquals, + record.vchunk_id}, + {metadata_key, EtcdHelper::TxnCompareKind::kValueEquals, current}}, + {{metadata_key, value}}); +} + +ErrorCode EtcdVChunkMetadataStore::Remove( + const VChunkMetadataRecord& record) { + if (connection_error_ != ErrorCode::OK) return connection_error_; + const auto metadata_key = MakePersistentRecordKey(namespace_prefix_, record); + const auto object_key = MakeObjectIndexKey(namespace_prefix_, record); + std::string current; + EtcdRevisionId revision = 0; + auto error = EtcdHelper::Get(metadata_key.data(), metadata_key.size(), + current, revision); + if (error == ErrorCode::ETCD_KEY_NOT_EXIST) return ErrorCode::OK; + if (error != ErrorCode::OK) return error; + return EtcdHelper::TxnCompareAndPut( + {{object_key, EtcdHelper::TxnCompareKind::kValueEquals, + record.vchunk_id}, + {metadata_key, EtcdHelper::TxnCompareKind::kValueEquals, current}}, + {}, {metadata_key, object_key}); +} + +tl::expected, ErrorCode> +EtcdVChunkMetadataStore::List() { + if (connection_error_ != ErrorCode::OK) { + return tl::unexpected(connection_error_); + } + const std::string begin = namespace_prefix_ + "/"; + const std::string end = namespace_prefix_ + "0"; + std::string json; + EtcdRevisionId revision = 0; + auto error = EtcdHelper::GetRangeAsJson(begin.data(), begin.size(), + end.data(), end.size(), 0, json, + revision); + if (error != ErrorCode::OK) return tl::unexpected(error); + + Json::Value root; + Json::CharReaderBuilder reader; + std::string errors; + std::istringstream stream(json); + if (!Json::parseFromStream(reader, stream, &root, &errors) || + !root.isArray()) { + return tl::unexpected(ErrorCode::INTERNAL_ERROR); + } + std::vector result; + for (const auto& item : root) { + if (!item.isObject() || !item["key"].isString() || + !item["value"].isString()) { + return tl::unexpected(ErrorCode::INTERNAL_ERROR); + } + const auto key = item["key"].asString(); + if (key.find(begin + "records/") != 0) continue; + auto bytes = HexDecode(item["value"].asString()); + if (!bytes) return tl::unexpected(bytes.error()); + auto record = DeserializeVChunkMetadata(*bytes, config_); + if (!record) return tl::unexpected(record.error()); + result.push_back(std::move(*record)); + } + return result; +} + +} // namespace mooncake diff --git a/mooncake-store/src/vchunk_metrics.cpp b/mooncake-store/src/vchunk_metrics.cpp new file mode 100644 index 0000000000..2d689d98fe --- /dev/null +++ b/mooncake-store/src/vchunk_metrics.cpp @@ -0,0 +1,71 @@ +#include "vchunk_metrics.h" + +#include + +namespace mooncake { + +void VChunkMetrics::Observe(VChunkOperation operation, bool success, + uint64_t latency_us) { + const auto index = static_cast(operation); + ++requests_[index]; + if (success) { + ++successes_[index]; + } + latency_us_[index].fetch_add(latency_us); +} + +void VChunkMetrics::SetStateCount(VChunkStatus state, uint64_t count) { + states_[static_cast(state)].store(count); +} + +void VChunkMetrics::ObserveLayout(const VChunkMetadataRecord& record) { + std::unordered_set segments; + for (const auto& slice : record.slices) { + segments.insert(slice.target_segment_name); + } + segment_participations_.fetch_add(segments.size()); + size_t bucket = 0; + switch (record.slice_size_level) { + case VCSliceSizeLevel::k4K: + bucket = 0; + break; + case VCSliceSizeLevel::k64K: + bucket = 1; + break; + case VCSliceSizeLevel::k256K: + bucket = 2; + break; + case VCSliceSizeLevel::k1M: + bucket = 3; + break; + } + slice_size_distribution_[bucket].fetch_add(record.slice_count); +} + +VChunkMetricsSnapshot VChunkMetrics::Snapshot() const { + VChunkMetricsSnapshot result; + for (size_t i = 0; i < requests_.size(); ++i) { + result.requests[i] = requests_[i].load(); + result.successes[i] = successes_[i].load(); + result.latency_us[i] = latency_us_[i].load(); + } + result.slices = slices_.load(); + result.segment_participations = segment_participations_.load(); + result.allocation_failures = allocation_failures_.load(); + result.transfer_failures = transfer_failures_.load(); + result.timeouts = timeouts_.load(); + result.retries = retries_.load(); + result.rollbacks = rollbacks_.load(); + result.metadata_bytes = metadata_bytes_.load(); + result.allocated_bytes = allocated_bytes_.load(); + for (size_t i = 0; i < slice_size_distribution_.size(); ++i) { + result.slice_size_distribution[i] = + slice_size_distribution_[i].load(); + } + for (size_t i = 0; i < states_.size(); ++i) { + result.states[i] = states_[i].load(); + } + return result; +} + +} // namespace mooncake diff --git a/mooncake-store/src/vchunk_transfer_engine.cpp b/mooncake-store/src/vchunk_transfer_engine.cpp new file mode 100644 index 0000000000..9029af37f0 --- /dev/null +++ b/mooncake-store/src/vchunk_transfer_engine.cpp @@ -0,0 +1,140 @@ +#include "vchunk_transfer_engine.h" + +#include +#include +#include + +namespace mooncake { +namespace { + +class BatchGuard { + public: + BatchGuard(TransferEngine& engine, size_t size) + : engine_(engine), id_(engine.allocateBatchID(size)) {} + ~BatchGuard() { + TryFree(); + } + BatchID id() const { return id_; } + bool TryFree() { + if (id_ == INVALID_BATCH_ID) return true; + if (!engine_.freeBatchID(id_).ok()) return false; + id_ = INVALID_BATCH_ID; + return true; + } + + private: + TransferEngine& engine_; + BatchID id_; +}; + +} // namespace + +tl::expected, ErrorCode> +BuildVChunkTransferRequests(const VChunkMetadataRecord& record, void* buffer, + size_t length, TransferRequest::OpCode opcode, + const VChunkSegmentResolver& resolve_segment) { + VChunkConfig validation_config; + validation_config.enabled = true; + if (!buffer || !resolve_segment || record.total_size != length || + record.slices.empty()) { + return tl::make_unexpected(ErrorCode::INVALID_PARAMS); + } + const auto validation = ValidateVChunkMetadata(record, validation_config); + if (validation != ErrorCode::OK) { + return tl::make_unexpected(validation); + } + std::unordered_map handles; + std::vector requests; + requests.reserve(record.slices.size()); + size_t logical_offset = 0; + for (const auto& slice : record.slices) { + auto it = handles.find(slice.target_segment_name); + if (it == handles.end()) { + auto handle = resolve_segment(slice.target_segment_name); + if (!handle) { + return tl::make_unexpected(handle.error()); + } + it = handles.emplace(slice.target_segment_name, *handle).first; + } + if (slice.logical_length > length - logical_offset) { + return tl::make_unexpected(ErrorCode::INVALID_PARAMS); + } + requests.push_back(TransferRequest{ + opcode, static_cast(buffer) + logical_offset, it->second, + slice.target_offset, slice.logical_length}); + logical_offset += slice.logical_length; + } + if (logical_offset != length) { + return tl::make_unexpected(ErrorCode::INVALID_PARAMS); + } + return requests; +} + +ErrorCode TransferEngineVChunkDataPlane::Write( + const VChunkMetadataRecord& record, const void* source, size_t length, + std::chrono::steady_clock::time_point deadline) { + return Transfer(record, const_cast(source), length, + TransferRequest::WRITE, deadline); +} + +ErrorCode TransferEngineVChunkDataPlane::Read( + const VChunkMetadataRecord& record, void* destination, size_t length, + std::chrono::steady_clock::time_point deadline) { + return Transfer(record, destination, length, TransferRequest::READ, + deadline); +} + +ErrorCode TransferEngineVChunkDataPlane::Transfer( + const VChunkMetadataRecord& record, void* buffer, size_t length, + TransferRequest::OpCode opcode, + std::chrono::steady_clock::time_point deadline) { + auto requests = BuildVChunkTransferRequests( + record, buffer, length, opcode, [this](const std::string& segment) { + const auto handle = engine_.openSegment(segment); + if (handle == static_cast(ERR_INVALID_ARGUMENT)) { + return tl::expected( + tl::make_unexpected(ErrorCode::SEGMENT_NOT_FOUND)); + } + return tl::expected(handle); + }); + if (!requests) { + return requests.error(); + } + + BatchGuard batch(engine_, requests->size()); + if (batch.id() == INVALID_BATCH_ID) { + return ErrorCode::TRANSFER_FAIL; + } + const bool submit_failed = + !engine_.submitTransfer(batch.id(), *requests).ok(); + if (submit_failed && batch.TryFree()) return ErrorCode::TRANSFER_FAIL; + bool deadline_exceeded = false; + for (;;) { + deadline_exceeded = deadline_exceeded || + std::chrono::steady_clock::now() >= deadline; + TransferStatus status{}; + if (!engine_.getBatchTransferStatus(batch.id(), status).ok()) { + // Do not release an in-flight batch: TransferEngine may still be + // using the caller's buffer. Keep draining until a terminal state. + if (batch.TryFree()) return ErrorCode::TRANSFER_FAIL; + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + continue; + } + if (status.s == TransferStatusEnum::COMPLETED) { + if (submit_failed) return ErrorCode::TRANSFER_FAIL; + if (deadline_exceeded) return ErrorCode::RPC_TIMEOUT; + return status.transferred_bytes == length ? ErrorCode::OK + : ErrorCode::TRANSFER_FAIL; + } + if (status.s == TransferStatusEnum::FAILED || + status.s == TransferStatusEnum::TIMEOUT || + status.s == TransferStatusEnum::CANCELED || + status.s == TransferStatusEnum::INVALID) { + return deadline_exceeded ? ErrorCode::RPC_TIMEOUT + : ErrorCode::TRANSFER_FAIL; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } +} + +} // namespace mooncake diff --git a/mooncake-store/tests/CMakeLists.txt b/mooncake-store/tests/CMakeLists.txt index 303be02254..1969fe0f42 100644 --- a/mooncake-store/tests/CMakeLists.txt +++ b/mooncake-store/tests/CMakeLists.txt @@ -37,6 +37,15 @@ add_store_test(buffer_allocator_test buffer_allocator_test.cpp) add_store_test(runtime_accelerator_test runtime_accelerator_test.cpp) add_store_test(registered_pinned_memory_test registered_pinned_memory_test.cpp) add_store_test(allocation_strategy_test allocation_strategy_test.cpp) +add_store_test(vchunk_config_test vchunk_config_test.cpp) +add_store_test(vchunk_metadata_test vchunk_metadata_test.cpp) +add_store_test(vchunk_allocation_strategy_test + vchunk_allocation_strategy_test.cpp) +add_store_test(vchunk_master_manager_test vchunk_master_manager_test.cpp) +add_store_test(vchunk_master_service_test vchunk_master_service_test.cpp) +add_store_test(vchunk_client_test vchunk_client_test.cpp) +add_store_test(vchunk_transfer_engine_test vchunk_transfer_engine_test.cpp) +add_store_test(vchunk_metadata_store_test vchunk_metadata_store_test.cpp) add_store_test(replica_selection_test replica_selection_test.cpp) add_test( NAME replica_selection_env_opt_in_test @@ -101,6 +110,8 @@ add_store_test(thread_pool_test thread_pool_test.cpp) add_store_test(transfer_task_test transfer_task_test.cpp) add_store_test(tenant_quota_test tenant_quota_test.cpp) add_store_test(tenant_id_test tenant_id_test.cpp) +add_store_test(slot_hash_test slot_hash_test.cpp) +add_store_test(partition_router_test partition_router_test.cpp) add_store_test(segment_test segment_test.cpp) add_store_test(offset_allocator_test offset_allocator_test.cpp) add_store_test(utils_test utils_test.cpp) diff --git a/mooncake-store/tests/client_integration_test.cpp b/mooncake-store/tests/client_integration_test.cpp index 6710096d87..6a2de62870 100644 --- a/mooncake-store/tests/client_integration_test.cpp +++ b/mooncake-store/tests/client_integration_test.cpp @@ -20,6 +20,7 @@ #include "allocator.h" #include "client_service.h" +#include "cvm/slot_hash.h" #include "types.h" #include "utils.h" #include "test_server_helpers.h" @@ -788,10 +789,11 @@ TEST_F(ClientIntegrationTest, BatchPutMixedGroupIdsThroughClient) { auto find_group_id_on_different_shard = [](const std::string& key) { static constexpr size_t kMetadataShardCountForTest = 1024; const size_t key_shard = - std::hash{}(key) % kMetadataShardCountForTest; + cvm::KeySlot(TenantId::Default(), key) % + kMetadataShardCountForTest; for (int i = 0; i < 10000; ++i) { std::string group_id = key + "_group_" + std::to_string(i); - if (std::hash{}(group_id) % + if (cvm::KeySlot(TenantId::Default(), group_id) % kMetadataShardCountForTest != key_shard) { return group_id; diff --git a/mooncake-store/tests/e2e/CMakeLists.txt b/mooncake-store/tests/e2e/CMakeLists.txt index 7f477bd227..2606c11cfe 100644 --- a/mooncake-store/tests/e2e/CMakeLists.txt +++ b/mooncake-store/tests/e2e/CMakeLists.txt @@ -92,6 +92,21 @@ target_link_libraries(storage_backend_e2e_test PUBLIC add_test(NAME storage_backend_e2e_test COMMAND storage_backend_e2e_test) +add_executable(gc_e2e_test gc_e2e_test.cpp) +target_include_directories(gc_e2e_test PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/.. +) +target_link_libraries(gc_e2e_test PUBLIC + mooncake_store + transfer_engine + cachelib_memory_allocator + glog + gtest + pthread + ${ETCD_WRAPPER_LIB} +) +add_test(NAME gc_e2e_test COMMAND gc_e2e_test) + add_executable(oplog_batch_e2e_test oplog_batch_e2e_test.cpp process_handler.cpp) target_link_libraries(oplog_batch_e2e_test PUBLIC glog gtest pthread) add_test(NAME oplog_batch_e2e_test COMMAND oplog_batch_e2e_test) diff --git a/mooncake-store/tests/e2e/gc_e2e_test.cpp b/mooncake-store/tests/e2e/gc_e2e_test.cpp new file mode 100644 index 0000000000..b39f03269b --- /dev/null +++ b/mooncake-store/tests/e2e/gc_e2e_test.cpp @@ -0,0 +1,535 @@ +// gc_e2e_test.cpp +// End-to-end integration tests for the explicit-delete-only SSD GC. +// +// Verifies the full pipeline that unit tests cannot cover: +// RealClient::remove -> master metadata erase +// -> FileStorage::MarkRemoved (tombstone) +// -> BucketStorageBackend GC compaction +// -> SSD bucket file reclamation +// +// Unlike storage_backend_e2e_test (which uses the Client base class + +// file-per-key backend), this suite uses RealClient with +// enable_ssd_offload=true so the BucketStorageBackend + FileStorage +// offload path is exercised, and remove goes through +// RealClient::remove_internal -> MarkRemoved. + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "client_buffer.h" +#include "real_client.h" +#include "test_server_helpers.h" +#include "types.h" + +DEFINE_string(protocol, "tcp", "Transfer protocol: rdma|tcp"); +DEFINE_string(device_name, "", "Device name to use, valid if protocol=rdma"); + +namespace mooncake { +namespace testing { + +namespace fs = std::filesystem; + +static constexpr size_t kMB = 1024ULL * 1024; + +// Count regular files with a given suffix in dir. +static int CountFilesWithSuffix(const fs::path& dir, + const std::string& suffix) { + int count = 0; + std::error_code ec; + for (auto& entry : fs::directory_iterator(dir, ec)) { + if (entry.is_regular_file()) { + auto name = entry.path().filename().string(); + if (name.size() >= suffix.size() && + name.compare(name.size() - suffix.size(), suffix.size(), + suffix) == 0) { + ++count; + } + } + } + return count; +} + +// List all .bucket file names (without directory) in dir. +static std::vector ListBucketFiles(const fs::path& dir) { + std::vector names; + std::error_code ec; + for (auto& entry : fs::directory_iterator(dir, ec)) { + if (entry.is_regular_file()) { + auto name = entry.path().filename().string(); + if (name.size() >= 6 && + name.compare(name.size() - 6, 6, ".bucket") == 0) { + names.push_back(name); + } + } + } + return names; +} + +// Read a key via RealClient::get_buffer into a std::string. Returns +// std::nullopt on failure. +static std::optional ReadKey( + const std::shared_ptr& client, const std::string& key) { + auto buf = client->get_buffer(key); + if (!buf) return std::nullopt; + return std::string(static_cast(buf->ptr()), buf->size()); +} + +class GCE2ETest : public ::testing::Test { + protected: + static void SetUpTestSuite() { + google::InitGoogleLogging("GCE2ETest"); + FLAGS_logtostderr = 1; + } + + static void TearDownTestSuite() { google::ShutdownGoogleLogging(); } + + void SetUp() override { + if (getenv("PROTOCOL")) FLAGS_protocol = getenv("PROTOCOL"); + if (getenv("DEVICE_NAME")) FLAGS_device_name = getenv("DEVICE_NAME"); + + tmp_dir_ = fs::temp_directory_path() / + ("mc_gc_e2e_" + std::to_string(::getpid())); + fs::create_directories(tmp_dir_); + + // Save and set the GC-required bucket backend env vars. + // eviction_policy=LRU keeps last_access_ns_ updated for GC candidate + // coldness; disable_ssd_eviction=true makes PrepareEviction a no-op + // so no live bucket is ever evicted. + saved_policy_ = GetEnvOpt("MOONCAKE_OFFLOAD_BUCKET_EVICTION_POLICY"); + setenv("MOONCAKE_OFFLOAD_BUCKET_EVICTION_POLICY", "lru", 1); + saved_disable_ = GetEnvOpt("MOONCAKE_OFFLOAD_DISABLE_SSD_EVICTION"); + setenv("MOONCAKE_OFFLOAD_DISABLE_SSD_EVICTION", "true", 1); + // Tighten GC so compaction runs after remove. Set interval long + // enough that GC doesn't fire during offload settlement (which + // could compact a bucket before remove creates a tombstone). + saved_gc_interval_ = GetEnvOpt("MOONCAKE_OFFLOAD_BUCKET_GC_INTERVAL_MS"); + setenv("MOONCAKE_OFFLOAD_BUCKET_GC_INTERVAL_MS", "15000", 1); + saved_gc_ratio_ = GetEnvOpt("MOONCAKE_OFFLOAD_BUCKET_GC_DELETED_RATIO"); + setenv("MOONCAKE_OFFLOAD_BUCKET_GC_DELETED_RATIO", "0.01", 1); + // Set bucket_keys_limit=1 so each offloaded key fills a bucket + // immediately. This ensures .bucket files are written on the first + // heartbeat after put. With limit=2, keys may sit in the ungrouped + // pool and no .bucket file is written. + saved_bucket_keys_limit_ = + GetEnvOpt("MOONCAKE_OFFLOAD_BUCKET_KEYS_LIMIT"); + setenv("MOONCAKE_OFFLOAD_BUCKET_KEYS_LIMIT", "1", 1); + } + + void TearDown() override { + if (real_client_) real_client_->tearDownAll(); + master_.Stop(); + easylog::set_min_severity(easylog::Severity::WARN); + + // Restore env. + RestoreEnv("MOONCAKE_OFFLOAD_BUCKET_EVICTION_POLICY", saved_policy_); + RestoreEnv("MOONCAKE_OFFLOAD_DISABLE_SSD_EVICTION", saved_disable_); + RestoreEnv("MOONCAKE_OFFLOAD_BUCKET_GC_INTERVAL_MS", saved_gc_interval_); + RestoreEnv("MOONCAKE_OFFLOAD_BUCKET_GC_DELETED_RATIO", saved_gc_ratio_); + RestoreEnv("MOONCAKE_OFFLOAD_BUCKET_KEYS_LIMIT", + saved_bucket_keys_limit_); + + std::error_code ec; + fs::remove_all(tmp_dir_, ec); + } + + static void RestoreEnv(const char* name, + const std::optional& saved) { + if (saved.has_value()) { + setenv(name, saved->c_str(), 1); + } else { + unsetenv(name); + } + } + + // Safely capture an env var as optional (getenv may return nullptr). + static std::optional GetEnvOpt(const char* name) { + const char* val = getenv(name); + if (val) return std::string(val); + return std::nullopt; + } + + bool StartMasterWithOffload() { + // Match production config: enable_offload=true, no root_fs_dir + // (master doesn't do disk caching; offload tasks are pushed to the + // client's FileStorage via heartbeat). Set a long lease TTL so + // objects aren't evicted before offload completes. + auto config = InProcMasterConfigBuilder() + .set_enable_offload(true) + .set_default_kv_lease_ttl(300000) + .build(); + return master_.Start(config); + } + + bool StartRealClient() { + real_client_ = RealClient::create(); + if (!real_client_) return false; + const std::string rdma_devices = + (FLAGS_protocol == "rdma") ? FLAGS_device_name : ""; + std::string ssd_path = tmp_dir_.string() + "/ssd_offload"; + fs::create_directories(ssd_path); + // Set MOONCAKE_OFFLOAD_FILE_STORAGE_PATH env var (same as production) + // so FileStorageConfig::FromEnvironment picks it up. This matches the + // production deployment pattern where the env var is set before + // launching the client. + setenv("MOONCAKE_OFFLOAD_FILE_STORAGE_PATH", ssd_path.c_str(), 1); + setenv("MOONCAKE_OFFLOAD_STORAGE_BACKEND_DESCRIPTOR", + "bucket_storage_backend", 1); + // enable_ssd_offload=true creates FileStorage + BucketStorageBackend. + int ret = real_client_->setup_real( + "localhost:17890", "P2PHANDSHAKE", + /*global_segment_size=*/512 * kMB, + /*local_buffer_size=*/256 * kMB, FLAGS_protocol, rdma_devices, + master_.master_address(), nullptr, + /*ipc_socket_path=*/"", + /*enable_ssd_offload=*/true, + /*ssd_offload_path=*/ssd_path, + /*tenant_id=*/"default"); + return ret == 0; + } + + // Put a key via RealClient and wait until it has been offloaded to the + // BucketStorageBackend (a .bucket file appears on SSD). Returns false on + // timeout. Waiting on memory reads is insufficient — offload is async + // (PutEnd queues, heartbeat drains) and MarkRemoved is a no-op until the + // key lands in object_bucket_map_. + bool PutAndWaitOffloaded(const std::string& key, + const std::string& value, + const fs::path& ssd_dir) { + std::span span(value.data(), value.size()); + ReplicateConfig config; + config.replica_num = 1; + if (real_client_->put(key, span, config) != 0) { + return false; + } + // Wait for offload: a .bucket file must appear in ssd_dir, AND the + // key must be readable via get_buffer (confirms data integrity). + // Heartbeat interval is 10s, so wait up to 40s. + for (int i = 0; i < 400; ++i) { // up to 40s + int buckets = CountFilesWithSuffix(ssd_dir, ".bucket"); + if (buckets > 0) { + auto got = ReadKey(real_client_, key); + if (got.has_value() && got.value() == value) return true; + } + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + return false; + } + + // After all keys are put and individually confirmed offloaded, wait an + // extra heartbeat cycle to ensure ALL keys have been drained from the + // offloading queue into object_bucket_map_. Without this, a key put + // after the first heartbeat may not yet be in object_bucket_map_ when + // MarkRemoved is called, making the tombstone a no-op. + void WaitForAllOffloadsSettled() { + // Wait less than gc_interval_ms (15s) so GC doesn't fire during + // settlement. Two heartbeat cycles (10s each) would be ideal, but + // 12s is enough for the 2nd heartbeat to drain remaining tasks + // while staying under the GC interval. + std::this_thread::sleep_for(std::chrono::seconds(12)); + } + + // Put multiple keys, then wait until ALL are offloaded (a .bucket file + // appears and each key is readable). Keys put before the next heartbeat + // are grouped into the same bucket (up to bucket_keys_limit). + bool PutBatchAndWaitOffloaded( + const std::vector>& kvs, + const fs::path& ssd_dir) { + ReplicateConfig config; + config.replica_num = 1; + for (const auto& [key, value] : kvs) { + std::span span(value.data(), value.size()); + if (real_client_->put(key, span, config) != 0) return false; + } + // Wait for offload of all keys: a .bucket file MUST appear (offload + // completed) AND each key must be readable via get_buffer. + // Heartbeat interval is 10s; with bucket_keys_limit=2, 2 keys fill + // a bucket on the first heartbeat after put. Wait up to 40s. + for (int i = 0; i < 400; ++i) { // up to 40s + if (CountFilesWithSuffix(ssd_dir, ".bucket") > 0) { + bool all_ok = true; + for (const auto& [key, value] : kvs) { + auto got = ReadKey(real_client_, key); + if (!got.has_value() || got.value() != value) { + all_ok = false; + break; + } + } + if (all_ok) return true; + } + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + return false; + } + + // Wait for GC compaction: detect by old bucket file(s) disappearing and + // new one(s) appearing. Returns the set of bucket file names before and + // after for caller verification. Polls up to 30s. + bool WaitForCompaction(const fs::path& ssd_dir, + const std::vector& buckets_before, + std::vector& buckets_after) { + for (int i = 0; i < 150; ++i) { // up to 30s + buckets_after = ListBucketFiles(ssd_dir); + // Compaction: at least one old bucket file gone, or set changed. + if (buckets_after != buckets_before) { + return true; + } + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + } + buckets_after = ListBucketFiles(ssd_dir); + return false; + } + + fs::path tmp_dir_; + InProcMaster master_; + std::shared_ptr real_client_; + std::optional saved_policy_; + std::optional saved_disable_; + std::optional saved_gc_interval_; + std::optional saved_gc_ratio_; + std::optional saved_bucket_keys_limit_; +}; + +// ------------------------------------------------------------------- +// Test 1: RemoveReclaimsSSDSpace +// +// Put 2 keys (each in its own bucket, bucket_keys_limit=1), remove k1. +// GC compaction must delete k1's (now-empty) bucket file. Validates: +// - k1's .bucket file is eventually deleted (space reclaimed) +// - k2 remains readable with correct data throughout +// - k1 stays gone +// ------------------------------------------------------------------- +TEST_F(GCE2ETest, RemoveReclaimsSSDSpace) { + ASSERT_TRUE(StartMasterWithOffload()); + ASSERT_TRUE(StartRealClient()); + + const std::string k1 = "gc_e2e_k1"; + const std::string k2 = "gc_e2e_k2"; + const std::string v1(4 * kMB, 'A'); + const std::string v2(4 * kMB, 'B'); + + fs::path ssd_dir = tmp_dir_ / "ssd_offload"; + ASSERT_TRUE(PutAndWaitOffloaded(k1, v1, ssd_dir)) + << "k1 offload timed out"; + ASSERT_TRUE(PutAndWaitOffloaded(k2, v2, ssd_dir)) + << "k2 offload timed out"; + + int buckets_before = CountFilesWithSuffix(ssd_dir, ".bucket"); + ASSERT_GT(buckets_before, 0) << "No bucket files after offload"; + + // Wait for all offload tasks to settle into object_bucket_map_. + WaitForAllOffloadsSettled(); + + // Snapshot bucket file names AFTER settle (all buckets written) and + // BEFORE remove. This is the baseline for detecting GC compaction. + auto bucket_files_before = ListBucketFiles(ssd_dir); + int buckets_after_settle = CountFilesWithSuffix(ssd_dir, ".bucket"); + + // Remove k1. GC should compact (delete k1's empty bucket file). + ASSERT_EQ(real_client_->remove(k1, /*force=*/true), 0); + + // Wait for GC: bucket file set should change (old file deleted, and/or + // new file written if compaction rewrote surviving keys). + bool reclaimed = false; + for (int i = 0; i < 150; ++i) { // up to 30s + // k2 must stay readable with correct data throughout GC. + auto got2 = ReadKey(real_client_, k2); + ASSERT_TRUE(got2.has_value()) + << "Surviving key k2 became unreadable during GC"; + ASSERT_EQ(got2.value(), v2) + << "Surviving key k2 data corrupted during GC"; + // k1 must stay gone. + auto got1 = ReadKey(real_client_, k1); + ASSERT_FALSE(got1.has_value()) + << "Removed key k1 became readable again"; + + // Detect: any file in before-set gone, OR count decreased. + auto bucket_files_now = ListBucketFiles(ssd_dir); + for (const auto& old_name : bucket_files_before) { + if (std::find(bucket_files_now.begin(), + bucket_files_now.end(), + old_name) == bucket_files_now.end()) { + reclaimed = true; + break; + } + } + int buckets_now = CountFilesWithSuffix(ssd_dir, ".bucket"); + if (!reclaimed && buckets_now < buckets_after_settle) { + reclaimed = true; + } + if (reclaimed) break; + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + } + + // Final checks. + auto got = ReadKey(real_client_, k2); + ASSERT_TRUE(got.has_value()); + EXPECT_EQ(got.value(), v2) << "Surviving key data corrupted after GC"; + + EXPECT_TRUE(reclaimed) << "GC did not reduce bucket file count within 30s"; +} + +// ------------------------------------------------------------------- +// Test 2: RemoveMiddleKeyPreservesSurvivors +// +// Put 2 keys, remove 1, wait for GC. The surviving key must remain +// readable with correct data. This is the core "don't lose un-removed +// keys" invariant. +// ------------------------------------------------------------------- +TEST_F(GCE2ETest, RemoveMiddleKeyPreservesSurvivors) { + ASSERT_TRUE(StartMasterWithOffload()); + ASSERT_TRUE(StartRealClient()); + + const std::string k1 = "gc_mid_k1"; + const std::string k2 = "gc_mid_k2"; + const std::string v1(4 * kMB, 'X'); + const std::string v2(4 * kMB, 'Y'); + + fs::path ssd_dir = tmp_dir_ / "ssd_offload"; + ASSERT_TRUE(PutAndWaitOffloaded(k1, v1, ssd_dir)) + << "k1 offload timed out"; + ASSERT_TRUE(PutAndWaitOffloaded(k2, v2, ssd_dir)) + << "k2 offload timed out"; + + int buckets_before = CountFilesWithSuffix(ssd_dir, ".bucket"); + ASSERT_GT(buckets_before, 0); + + // Wait for all offload tasks to settle into object_bucket_map_. + WaitForAllOffloadsSettled(); + + // Snapshot bucket file names AFTER settle (all buckets written) and + // BEFORE remove. This is the baseline for detecting GC compaction. + auto bucket_files_before = ListBucketFiles(ssd_dir); + // Re-count after settle — more buckets may have appeared. + int buckets_after_settle = CountFilesWithSuffix(ssd_dir, ".bucket"); + + // Remove k2 (force=true to bypass lease). + ASSERT_EQ(real_client_->remove(k2, /*force=*/true), 0); + + // Wait for GC: bucket file set should change. + bool compacted = false; + for (int i = 0; i < 150; ++i) { + auto got1 = ReadKey(real_client_, k1); + if (got1.has_value() && got1.value() == v1) { + auto bucket_files_now = ListBucketFiles(ssd_dir); + // Detect: any file in before-set gone, OR count decreased. + for (const auto& old_name : bucket_files_before) { + if (std::find(bucket_files_now.begin(), + bucket_files_now.end(), + old_name) == bucket_files_now.end()) { + compacted = true; + break; + } + } + int buckets_now = CountFilesWithSuffix(ssd_dir, ".bucket"); + if (!compacted && buckets_now < buckets_after_settle) { + compacted = true; + } + if (compacted) break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + } + + // k1 must survive with correct data. + auto got1 = ReadKey(real_client_, k1); + ASSERT_TRUE(got1.has_value()); + EXPECT_EQ(got1.value(), v1) << "k1 data corrupted after GC"; + + // k2 must remain gone. + auto got2 = ReadKey(real_client_, k2); + EXPECT_FALSE(got2.has_value()) + << "Removed key k2 should not be readable"; + + EXPECT_TRUE(compacted) << "GC compaction not detected within 30s"; +} + +// ------------------------------------------------------------------- +// Test 3: BatchRemoveMixedExistingAndAbsent +// +// BatchRemove with a non-existent key mixed in: the existing key should +// be tombstoned + GC'd, the non-existent one ignored. +// ------------------------------------------------------------------- +TEST_F(GCE2ETest, BatchRemoveMixedExistingAndAbsent) { + ASSERT_TRUE(StartMasterWithOffload()); + ASSERT_TRUE(StartRealClient()); + + const std::string k1 = "gc_batch_k1"; + const std::string k2 = "gc_batch_k2"; + const std::string v1(4 * kMB, 'Q'); + const std::string v2(4 * kMB, 'R'); + + fs::path ssd_dir = tmp_dir_ / "ssd_offload"; + ASSERT_TRUE(PutAndWaitOffloaded(k1, v1, ssd_dir)) + << "k1 offload timed out"; + ASSERT_TRUE(PutAndWaitOffloaded(k2, v2, ssd_dir)) + << "k2 offload timed out"; + + int buckets_before = CountFilesWithSuffix(ssd_dir, ".bucket"); + ASSERT_GT(buckets_before, 0); + + // Wait for all offload tasks to settle into object_bucket_map_. + WaitForAllOffloadsSettled(); + + // Snapshot bucket file names AFTER settle (all buckets written) and + // BEFORE remove. + auto bucket_files_before = ListBucketFiles(ssd_dir); + int buckets_after_settle = CountFilesWithSuffix(ssd_dir, ".bucket"); + + // Batch remove: k1 exists, k_absent does not. force=true bypasses lease. + std::vector keys{k1, "gc_batch_absent"}; + auto results = real_client_->batchRemove(keys, /*force=*/true); + ASSERT_EQ(results.size(), 2u); + + // Wait for GC: bucket file set should change. + bool compacted = false; + for (int i = 0; i < 150; ++i) { + auto got2 = ReadKey(real_client_, k2); + if (got2.has_value() && got2.value() == v2) { + auto bucket_files_now = ListBucketFiles(ssd_dir); + for (const auto& old_name : bucket_files_before) { + if (std::find(bucket_files_now.begin(), + bucket_files_now.end(), + old_name) == bucket_files_now.end()) { + compacted = true; + break; + } + } + int buckets_now = CountFilesWithSuffix(ssd_dir, ".bucket"); + if (!compacted && buckets_now < buckets_after_settle) { + compacted = true; + } + if (compacted) break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + } + + // k1 must be gone. + auto got1 = ReadKey(real_client_, k1); + EXPECT_FALSE(got1.has_value()) + << "Batch-removed key k1 should not be readable"; + + // k2 must survive. + auto got2 = ReadKey(real_client_, k2); + ASSERT_TRUE(got2.has_value()); + EXPECT_EQ(got2.value(), v2) << "k2 data corrupted after GC"; + + EXPECT_TRUE(compacted) << "GC compaction not detected within 30s"; +} + +} // namespace testing +} // namespace mooncake + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + gflags::ParseCommandLineFlags(&argc, &argv, false); + return RUN_ALL_TESTS(); +} diff --git a/mooncake-store/tests/ha/master_service_ha_test.cpp b/mooncake-store/tests/ha/master_service_ha_test.cpp index 265398ff47..4fadd69554 100644 --- a/mooncake-store/tests/ha/master_service_ha_test.cpp +++ b/mooncake-store/tests/ha/master_service_ha_test.cpp @@ -1600,7 +1600,7 @@ TEST_F(MasterServiceBatchRecordE2ETest, PromotionCatchesUpToDurablePrefix) { HotStandbyService standby(standby_config); standby.SetCatchUpBatchKvBackendForTesting(backend); - auto start_err = standby.Start("", "", cluster_id); + auto start_err = standby.Start({}, "", cluster_id); ASSERT_EQ(ErrorCode::OK, start_err); ASSERT_EQ(StandbyState::WATCHING, standby.GetState()); diff --git a/mooncake-store/tests/ha/standby/hot_standby_service_test.cpp b/mooncake-store/tests/ha/standby/hot_standby_service_test.cpp index 317a48d799..d6d1784285 100644 --- a/mooncake-store/tests/ha/standby/hot_standby_service_test.cpp +++ b/mooncake-store/tests/ha/standby/hot_standby_service_test.cpp @@ -104,7 +104,7 @@ std::unique_ptr CreateSnapshotOnlyReadyStandby( service->SetSnapshotProvider(std::make_unique( std::optional(snapshot))); - EXPECT_EQ(ErrorCode::OK, service->Start("", "", cluster_id)); + EXPECT_EQ(ErrorCode::OK, service->Start({}, "", cluster_id)); EXPECT_EQ(StandbyState::WATCHING, service->GetState()); return service; } @@ -280,7 +280,7 @@ TEST_F(HotStandbyServiceTest, TestPromoteAndExportSnapshot_FinalCatchUp) { service_->SetSnapshotProvider(std::make_unique( std::optional(snapshot))); - ASSERT_EQ(ErrorCode::OK, service_->Start("", "", cluster_id_)); + ASSERT_EQ(ErrorCode::OK, service_->Start({}, "", cluster_id_)); EXPECT_EQ(StandbyState::WATCHING, service_->GetState()); EXPECT_EQ(10u, service_->GetLatestAppliedSequenceId()); @@ -347,7 +347,7 @@ TEST_F(HotStandbyServiceTest, TestStart_SnapshotOnlyWithSnapshot) { service_->SetSnapshotProvider(std::make_unique( std::optional(snapshot))); - auto err = service_->Start("", "", cluster_id_); + auto err = service_->Start({}, "", cluster_id_); EXPECT_EQ(ErrorCode::OK, err); EXPECT_EQ(StandbyState::WATCHING, service_->GetState()); EXPECT_EQ(1u, service_->GetMetadataCount()); @@ -375,7 +375,7 @@ TEST_F(HotStandbyServiceTest, std::make_unique(std::optional( MakeSnapshot("20260330_120000_000", 42, "key-old", 4096)))); - ASSERT_EQ(ErrorCode::OK, service_->Start("", "", cluster_id_)); + ASSERT_EQ(ErrorCode::OK, service_->Start({}, "", cluster_id_)); EXPECT_EQ(StandbyState::WATCHING, service_->GetState()); EXPECT_EQ(42u, service_->GetLatestAppliedSequenceId()); EXPECT_EQ(1u, service_->GetMetadataCount()); @@ -385,7 +385,7 @@ TEST_F(HotStandbyServiceTest, std::make_unique(std::optional( MakeSnapshot("20260330_121500_000", 84, "key-new", 8192)))); - ASSERT_EQ(ErrorCode::OK, service_->Start("", "", cluster_id_)); + ASSERT_EQ(ErrorCode::OK, service_->Start({}, "", cluster_id_)); EXPECT_EQ(StandbyState::WATCHING, service_->GetState()); EXPECT_EQ(84u, service_->GetLatestAppliedSequenceId()); EXPECT_EQ(1u, service_->GetMetadataCount()); @@ -406,7 +406,7 @@ TEST_F(HotStandbyServiceTest, TestStart_SnapshotOnlyWhenProviderFails) { service_->SetSnapshotProvider(std::make_unique( tl::make_unexpected(ErrorCode::PERSISTENT_FAIL))); - auto err = service_->Start("", "", cluster_id_); + auto err = service_->Start({}, "", cluster_id_); EXPECT_EQ(ErrorCode::PERSISTENT_FAIL, err); EXPECT_EQ(StandbyState::FAILED, service_->GetState()); } @@ -457,7 +457,7 @@ TEST_F(HotStandbyServiceTest, TestExportStandbySnapshot_Empty) { service_->SetSnapshotProvider(std::make_unique( std::optional(LoadedSnapshot{}))); - EXPECT_EQ(ErrorCode::OK, service_->Start("", "", cluster_id_)); + EXPECT_EQ(ErrorCode::OK, service_->Start({}, "", cluster_id_)); EXPECT_EQ(StandbyState::WATCHING, service_->GetState()); StandbySnapshot snapshot; @@ -811,7 +811,7 @@ TEST_F(PromotionCatchUpTest, UsesDurablePrefixLastSeqAsCatchUpTarget) { batch_backend_->Put(BuildBatchRecordKey(cluster_id_, 1), EncodeOpLogBatchRecord(MakeBatch(1, 1, 2)))); - auto err = service_->Start("", oplog_endpoints_, cluster_id_); + auto err = service_->Start({}, oplog_endpoints_, cluster_id_); if (err != ErrorCode::OK) { GTEST_SKIP() << "Service could not reach WATCHING state; " "skipping promotion test"; @@ -836,7 +836,7 @@ TEST_F(PromotionCatchUpTest, RetriesTransientDurablePrefixReadFailure) { service_->SetCatchUpBatchKvBackendForTesting(batch_backend); ASSERT_EQ(ErrorCode::OK, - service_->Start("", oplog_endpoints_, cluster_id_)); + service_->Start({}, oplog_endpoints_, cluster_id_)); batch_backend->FailNextGet(ErrorCode::ETCD_OPERATION_ERROR); StandbySnapshot out; @@ -846,7 +846,7 @@ TEST_F(PromotionCatchUpTest, RetriesTransientDurablePrefixReadFailure) { TEST_F(PromotionCatchUpTest, MissingDurablePrefixPromotesAtSequenceZero) { ASSERT_EQ(ErrorCode::OK, - service_->Start("", oplog_endpoints_, cluster_id_)); + service_->Start({}, oplog_endpoints_, cluster_id_)); ASSERT_EQ(StandbyState::WATCHING, service_->GetState()); StandbySnapshot out; ASSERT_EQ(ErrorCode::OK, service_->PromoteAndExportSnapshot(out)); @@ -861,7 +861,7 @@ TEST_F(PromotionCatchUpTest, MissingDurablePrefixRejectsNonzeroSequence) { std::optional(MakeSnapshot("baseline", 1, "key", 1)))); ASSERT_EQ(ErrorCode::OK, - service_->Start("", oplog_endpoints_, cluster_id_)); + service_->Start({}, oplog_endpoints_, cluster_id_)); ASSERT_EQ(StandbyState::WATCHING, service_->GetState()); StandbySnapshot out; EXPECT_EQ(ErrorCode::INCOMPLETE_OPLOG_CATCH_UP, @@ -871,7 +871,7 @@ TEST_F(PromotionCatchUpTest, MissingDurablePrefixRejectsNonzeroSequence) { TEST_F(PromotionCatchUpTest, CatchesUpPrefixThatAppearsBeforePromotion) { ASSERT_EQ(ErrorCode::OK, - service_->Start("", oplog_endpoints_, cluster_id_)); + service_->Start({}, oplog_endpoints_, cluster_id_)); ASSERT_EQ(StandbyState::WATCHING, service_->GetState()); ASSERT_EQ(ErrorCode::OK, batch_backend_->Put( @@ -902,7 +902,7 @@ TEST_F(PromotionCatchUpTest, PaginatesBatchRecords) { } service_->SetCatchUpBatchKvBackendForTesting(batch_backend); - auto err = service_->Start("", oplog_endpoints_, cluster_id_); + auto err = service_->Start({}, oplog_endpoints_, cluster_id_); if (err != ErrorCode::OK) { GTEST_SKIP() << "Service could not reach WATCHING state; " "skipping promotion test"; @@ -915,7 +915,7 @@ TEST_F(PromotionCatchUpTest, PaginatesBatchRecords) { TEST_F(PromotionCatchUpTest, FailsPromotionWhenDurablePrefixUnreadable) { ASSERT_EQ(ErrorCode::OK, - service_->Start("", oplog_endpoints_, cluster_id_)); + service_->Start({}, oplog_endpoints_, cluster_id_)); batch_backend_->SetGetError(ErrorCode::PERSISTENT_FAIL); StandbySnapshot out; @@ -926,7 +926,7 @@ TEST_F(PromotionCatchUpTest, FailsPromotionWhenDurablePrefixUnreadable) { TEST_F(PromotionCatchUpTest, FailsPromotionWhenTargetBatchUnreadable) { ASSERT_EQ(ErrorCode::OK, - service_->Start("", oplog_endpoints_, cluster_id_)); + service_->Start({}, oplog_endpoints_, cluster_id_)); ASSERT_EQ(ErrorCode::OK, batch_backend_->Put( BuildDurablePrefixKey(cluster_id_), diff --git a/mooncake-store/tests/ha/standby/hot_standby_snapshot_bootstrap_test.cpp b/mooncake-store/tests/ha/standby/hot_standby_snapshot_bootstrap_test.cpp index c9a425b563..75b3224b07 100644 --- a/mooncake-store/tests/ha/standby/hot_standby_snapshot_bootstrap_test.cpp +++ b/mooncake-store/tests/ha/standby/hot_standby_snapshot_bootstrap_test.cpp @@ -103,7 +103,7 @@ TEST_P(HotStandbySnapshotBootstrapTest, HotStandbyService service(MakeSnapshotOnlyConfig()); service.SetSnapshotProvider(std::move(provider.value())); - ASSERT_EQ(ErrorCode::OK, service.Start("", "", cluster_id_)); + ASSERT_EQ(ErrorCode::OK, service.Start({}, "", cluster_id_)); EXPECT_EQ(StandbyState::WATCHING, service.GetState()); EXPECT_EQ(1u, service.GetMetadataCount()); EXPECT_EQ(descriptor_.last_included_seq, @@ -131,7 +131,7 @@ TEST_P(HotStandbySnapshotBootstrapTest, HotStandbyService service(MakeSnapshotOnlyConfig()); service.SetSnapshotProvider(std::move(provider.value())); - ASSERT_EQ(ErrorCode::OK, service.Start("", "", cluster_id_)); + ASSERT_EQ(ErrorCode::OK, service.Start({}, "", cluster_id_)); EXPECT_EQ(StandbyState::WATCHING, service.GetState()); EXPECT_EQ(0u, service.GetMetadataCount()); EXPECT_EQ(0u, service.GetLatestAppliedSequenceId()); @@ -160,7 +160,7 @@ TEST(StandbyControllerTest, PromoteStandbyReturnsStartFailure) { auto controller = ha::CreateStandbyController(spec, config); ASSERT_NE(controller, nullptr); EXPECT_EQ(ErrorCode::INVALID_PARAMS, - controller->StartStandby(std::nullopt)); + controller->StartStandby({})); EXPECT_EQ(ErrorCode::INVALID_PARAMS, controller->PromoteStandby()); } @@ -212,7 +212,7 @@ TEST(StandbyControllerTest, // After failed start EXPECT_EQ(ErrorCode::INVALID_PARAMS, - controller->StartStandby(std::nullopt)); + controller->StartStandby({})); result = controller->PromoteStandbyAndExport(); EXPECT_FALSE(result.has_value()); EXPECT_EQ(ErrorCode::INVALID_PARAMS, result.error()); @@ -236,7 +236,7 @@ TEST_P(HotStandbySnapshotBootstrapTest, auto controller = ha::CreateStandbyController(spec, config); ASSERT_NE(controller, nullptr); - EXPECT_EQ(ErrorCode::OK, controller->StartStandby(std::nullopt)); + EXPECT_EQ(ErrorCode::OK, controller->StartStandby({})); auto result = controller->PromoteStandbyAndExport(); ASSERT_TRUE(result.has_value()) << toString(result.error()); diff --git a/mooncake-store/tests/master_service_test.cpp b/mooncake-store/tests/master_service_test.cpp index bc3614949a..cd1e193e67 100644 --- a/mooncake-store/tests/master_service_test.cpp +++ b/mooncake-store/tests/master_service_test.cpp @@ -199,10 +199,11 @@ class MasterServiceTest : public ::testing::Test { std::string FindGroupIdOnDifferentShard(const std::string& key) const { static constexpr size_t kMetadataShardCountForTest = 1024; const size_t key_shard = - std::hash{}(key) % kMetadataShardCountForTest; + cvm::KeySlot(TenantId::Default(), key) % + kMetadataShardCountForTest; for (int i = 0; i < 10000; ++i) { std::string group_id = key + "_group_" + std::to_string(i); - if (std::hash{}(group_id) % + if (cvm::KeySlot(TenantId::Default(), group_id) % kMetadataShardCountForTest != key_shard) { return group_id; @@ -1043,8 +1044,8 @@ TEST_F(MasterServiceTest, GroupRoutingIsTenantScopedForSameUserKey) { std::string group_b; for (int i = 0; i < 10000; ++i) { group_b = key + "_tenant_b_group_" + std::to_string(i); - if (std::hash{}(group_b) % 1024 != - std::hash{}(group_a) % 1024) { + if (cvm::KeySlot(TenantId::Default(), group_b) % 1024 != + cvm::KeySlot(TenantId::Default(), group_a) % 1024) { break; } } @@ -1325,8 +1326,8 @@ TEST_F(MasterServiceTest, std::string group_b; for (int i = 0; i < 10000; ++i) { group_b = key + "_other_group_" + std::to_string(i); - if (std::hash{}(group_b) % 1024 != - std::hash{}(group_a) % 1024) { + if (cvm::KeySlot(TenantId::Default(), group_b) % 1024 != + cvm::KeySlot(TenantId::Default(), group_a) % 1024) { break; } } diff --git a/mooncake-store/tests/partition_router_test.cpp b/mooncake-store/tests/partition_router_test.cpp new file mode 100644 index 0000000000..54ffb1f50d --- /dev/null +++ b/mooncake-store/tests/partition_router_test.cpp @@ -0,0 +1,93 @@ +#include "partition/partition_router.h" + +#include + +#include +#include + +#include "cvm/slot_hash.h" +#include "tenant_id.h" + +namespace mooncake::test { +namespace { + +cvm::SlotOwner MakeOwner(uint16_t slot, const std::string& primary) { + cvm::SlotOwner owner; + owner.slot = slot; + owner.primary_master_id = primary; + owner.state = static_cast(cvm::SlotState::kStable); + return owner; +} + +TEST(PartitionRouterTest, LoadAndResolve) { + partition::PartitionRouter router; + router.LoadSlotOwners({MakeOwner(10, "m-a"), MakeOwner(20, "m-b")}); + + EXPECT_EQ(router.Size(), 2u); + ASSERT_TRUE(router.ResolveSubmaster(10).has_value()); + EXPECT_EQ(*router.ResolveSubmaster(10), "m-a"); + ASSERT_TRUE(router.ResolveSubmaster(20).has_value()); + EXPECT_EQ(*router.ResolveSubmaster(20), "m-b"); +} + +TEST(PartitionRouterTest, MissingSlotReturnsNullopt) { + partition::PartitionRouter router; + router.LoadSlotOwners({MakeOwner(10, "m-a")}); + EXPECT_FALSE(router.ResolveSubmaster(11).has_value()); +} + +TEST(PartitionRouterTest, SkipsEmptyPrimary) { + partition::PartitionRouter router; + router.LoadSlotOwners({MakeOwner(10, ""), MakeOwner(11, "m-b")}); + + EXPECT_EQ(router.Size(), 1u); + EXPECT_FALSE(router.ResolveSubmaster(10).has_value()); + EXPECT_TRUE(router.ResolveSubmaster(11).has_value()); +} + +TEST(PartitionRouterTest, SkipsMigratingSlotUntilOwnerIsStable) { + auto migrating = MakeOwner(10, "m-a"); + migrating.state = static_cast(cvm::SlotState::kMigrating); + migrating.migrating_to_master_id = "m-b"; + + partition::PartitionRouter router; + router.LoadSlotOwners({migrating, MakeOwner(11, "m-b")}); + + EXPECT_FALSE(router.ResolveSubmaster(10).has_value()); + EXPECT_EQ(router.ResolveSubmaster(11), "m-b"); +} + +TEST(PartitionRouterTest, OverwritesOnReload) { + partition::PartitionRouter router; + router.LoadSlotOwners({MakeOwner(10, "m-a")}); + router.LoadSlotOwners({MakeOwner(10, "m-b"), MakeOwner(20, "m-c")}); + + EXPECT_EQ(router.Size(), 2u); + ASSERT_TRUE(router.ResolveSubmaster(10).has_value()); + EXPECT_EQ(*router.ResolveSubmaster(10), "m-b"); + EXPECT_TRUE(router.ResolveSubmaster(20).has_value()); +} + +TEST(PartitionRouterTest, RouteUsesKeySlot) { + partition::PartitionRouter router; + const std::string key = "route-me"; + const TenantId tenant("tenant-r"); + const uint16_t slot = cvm::KeySlot(tenant, key); + router.LoadSlotOwners({MakeOwner(slot, "m-target")}); + + ASSERT_TRUE(router.Route(tenant, key).has_value()); + EXPECT_EQ(*router.Route(tenant, key), "m-target"); +} + +TEST(PartitionRouterTest, ClearAndSize) { + partition::PartitionRouter router; + router.LoadSlotOwners({MakeOwner(10, "m-a"), MakeOwner(20, "m-b")}); + EXPECT_EQ(router.Size(), 2u); + + router.Clear(); + EXPECT_EQ(router.Size(), 0u); + EXPECT_FALSE(router.ResolveSubmaster(10).has_value()); +} + +} // namespace +} // namespace mooncake::test diff --git a/mooncake-store/tests/promotion_on_hit_test.cpp b/mooncake-store/tests/promotion_on_hit_test.cpp index eb496cfde1..751c8857db 100644 --- a/mooncake-store/tests/promotion_on_hit_test.cpp +++ b/mooncake-store/tests/promotion_on_hit_test.cpp @@ -752,11 +752,11 @@ TEST_F(PromotionOnHitTest, QueueLimitRejectsBeyondCap) { // Find two keys that hash to the same shard. MasterService:: // getShardIndex is private but the formula is deterministic - // (std::hash{}(key) % kNumShards), so we can mirror - // it here. kNumShards=1024 (master_service.h:889). + // (cvm::KeySlot(default_tenant, key) % kNumShards), so we can mirror + // it here. kNumShards=1024. constexpr size_t kNumShardsLocal = 1024; auto shard_of = [](const std::string& k) { - return std::hash{}(k) % kNumShardsLocal; + return cvm::KeySlot(TenantId::Default(), k) % kNumShardsLocal; }; const std::string k1 = "qlim_first"; std::string k2; @@ -986,7 +986,7 @@ TEST_F(PromotionOnHitTest, QueueLimitRejectsCrossShard) { // independently). With the global counter, only the first goes in. constexpr size_t kNumShardsLocal = 1024; auto shard_of = [](const std::string& k) { - return std::hash{}(k) % kNumShardsLocal; + return cvm::KeySlot(TenantId::Default(), k) % kNumShardsLocal; }; const std::string k1 = "xshard_first"; std::string k2; diff --git a/mooncake-store/tests/slot_hash_test.cpp b/mooncake-store/tests/slot_hash_test.cpp new file mode 100644 index 0000000000..183145200d --- /dev/null +++ b/mooncake-store/tests/slot_hash_test.cpp @@ -0,0 +1,145 @@ +#include "cvm/slot_hash.h" + +#include + +#include +#include +#include + +#include "crc32c.h" +#include "partition/kv_hash_map.h" +#include "tenant_id.h" + +namespace mooncake::test { +namespace { + +TEST(SlotHashTest, SlotOfUsesLow14Bits) { + EXPECT_EQ(cvm::SlotOf(0u), 0u); + EXPECT_EQ(cvm::SlotOf(1u), 1u); + EXPECT_EQ(cvm::SlotOf(cvm::kSlotCount - 1), cvm::kSlotCount - 1); + // 第 15 位及以上被丢弃：16384 (2^14) 映射回 0。 + EXPECT_EQ(cvm::SlotOf(cvm::kSlotCount), 0u); + EXPECT_EQ(cvm::SlotOf(0xFFFFFFFFu), cvm::kSlotMask); +} + +TEST(SlotHashTest, KeySlotInRange) { + const std::vector keys = {"", "a", "hello", "key-123", + "a-longer-key-for-coverage"}; + for (const auto& key : keys) { + EXPECT_LT(cvm::KeySlot(TenantId::Default(), key), cvm::kSlotCount); + EXPECT_LT(cvm::KeySlot(TenantId("tenant-a"), key), cvm::kSlotCount); + } +} + +TEST(SlotHashTest, KeySlotDeterministic) { + const std::string key = "deterministic-key"; + const TenantId tenant("tenant-x"); + const uint16_t first = cvm::KeySlot(tenant, key); + for (int i = 0; i < 100; ++i) { + EXPECT_EQ(first, cvm::KeySlot(tenant, key)); + } +} + +namespace { +uint16_t ManualDefaultKeySlot(const std::string& key) { + Crc32c crc; + crc.Extend(key.data(), key.size()); + return cvm::SlotOf(crc.Final()); +} + +uint16_t ManualScopedKeySlot(const std::string& tenant, + const std::string& key) { + Crc32c crc; + crc.Extend(tenant.data(), tenant.size()); + constexpr char kSeparator = '\0'; + crc.Extend(&kSeparator, 1); + crc.Extend(key.data(), key.size()); + return cvm::SlotOf(crc.Final()); +} +} // namespace + +TEST(SlotHashTest, KeySlotMatchesManualCrc) { + const std::string key = "scope-me"; + // 默认 tenant：slot = hash(user_key)。 + EXPECT_EQ(cvm::KeySlot(TenantId::Default(), key), + ManualDefaultKeySlot(key)); + // 非默认 tenant：slot = hash(tenant + '\0' + user_key)。 + EXPECT_EQ(cvm::KeySlot(TenantId("tenant-a"), key), + ManualScopedKeySlot("tenant-a", key)); +} + +TEST(SlotHashTest, VNodePositionInRangeAndDeterministic) { + const std::string master = "master-1:10001"; + for (uint16_t vnode = 0; vnode < cvm::kVnodeCount; ++vnode) { + const uint16_t pos = cvm::VNodePosition(master, vnode); + EXPECT_LT(pos, cvm::kSlotCount); + EXPECT_EQ(pos, cvm::VNodePosition(master, vnode)); + } +} + +TEST(SlotHashTest, KvHashMapMatchesKeySlot) { + // client 侧路由（KvHashMap）与 submaster 侧校验（KeySlot）必须用同一份 + // 哈希，否则 key 会路由到错误的 submaster。 + const std::string key = "consistency-key"; + const TenantId tenant("tenant-c"); + EXPECT_EQ(partition::KvHashMap::Compute(tenant, key), + cvm::KeySlot(tenant, key)); +} + +TEST(SlotHashTest, ResolveOwnedSlotsOnRingSingleMaster) { + // n=1 时退化：本机拥有全部 slot。 + const std::vector ids = {"m-a"}; + const auto slots = cvm::ResolveOwnedSlotsOnRing(ids, "m-a"); + EXPECT_EQ(slots.size(), static_cast(cvm::kSlotCount)); +} + +TEST(SlotHashTest, ResolveOwnedSlotsOnRingCoversAllSlots) { + const std::vector ids = {"m-a", "m-b", "m-c"}; + std::vector covered(cvm::kSlotCount, false); + size_t total = 0; + for (const auto& id : ids) { + const auto slots = cvm::ResolveOwnedSlotsOnRing(ids, id); + for (uint16_t s : slots) { + EXPECT_FALSE(covered[s]) << "slot " << s << " owned by >1 master"; + covered[s] = true; + ++total; + } + } + // 每个 slot 恰好归属一个 primary（不重叠 + 全覆盖）。 + EXPECT_EQ(total, static_cast(cvm::kSlotCount)); + for (size_t s = 0; s < cvm::kSlotCount; ++s) { + EXPECT_TRUE(covered[s]) << "slot " << s << " has no owner"; + } +} + +TEST(SlotHashTest, ResolveOwnedSlotsOnRingStableOnRemoval) { + // 一致性哈希环核心性质：移除一个 primary 后，其余 primary 原有 slot 不 + // 丢（只增不减），仅被移除 primary 覆盖的 slot 发生重分配——即「非全员 + // 平移」。 + const std::vector ids3 = {"m-a", "m-b", "m-c"}; + const auto a = cvm::ResolveOwnedSlotsOnRing(ids3, "m-a"); + const auto b = cvm::ResolveOwnedSlotsOnRing(ids3, "m-b"); + + const std::vector ids2 = {"m-a", "m-b"}; + const auto a2 = cvm::ResolveOwnedSlotsOnRing(ids2, "m-a"); + const auto b2 = cvm::ResolveOwnedSlotsOnRing(ids2, "m-b"); + + const std::set a_set(a.begin(), a.end()); + const std::set a2_set(a2.begin(), a2.end()); + const std::set b_set(b.begin(), b.end()); + const std::set b2_set(b2.begin(), b2.end()); + + for (uint16_t s : a_set) { + EXPECT_TRUE(a2_set.count(s)) << "a lost slot " << s << " on removal"; + } + for (uint16_t s : b_set) { + EXPECT_TRUE(b2_set.count(s)) << "b lost slot " << s << " on removal"; + } + // 移除后 a/b 覆盖全量 slot，且总量较移除前增加（接管了被移除者的 slot）。 + EXPECT_EQ(a2_set.size() + b2_set.size(), + static_cast(cvm::kSlotCount)); + EXPECT_GT(a2_set.size() + b2_set.size(), a_set.size() + b_set.size()); +} + +} // namespace +} // namespace mooncake::test diff --git a/mooncake-store/tests/storage_backend_test.cpp b/mooncake-store/tests/storage_backend_test.cpp index 1f3e1ae246..77acec964d 100644 --- a/mooncake-store/tests/storage_backend_test.cpp +++ b/mooncake-store/tests/storage_backend_test.cpp @@ -2821,9 +2821,319 @@ TEST_F(StorageBackendTest, BucketStorageBackend_ConcurrentReadWriteDelete) { } //----------------------------------------------------------------------------- -// Tests for FileRecord key tracking and eviction return values +// Explicit-delete-only GC tests (tombstone + compaction) //----------------------------------------------------------------------------- +TEST_F(StorageBackendTest, BucketStorageBackend_MarkRemovedHidesKey) { + FileStorageConfig config; + config.storage_filepath = data_path; + BucketBackendConfig bucket_config; + BucketStorageBackend storage_backend(config, bucket_config); + ASSERT_TRUE(storage_backend.Init()); + + std::string k1 = "mark_k1"; + std::string k2 = "mark_k2"; + std::string v1 = "value1"; + std::string v2 = "value2"; + + std::unordered_map> batch; + auto buf1 = std::make_unique(v1.size()); + auto buf2 = std::make_unique(v2.size()); + std::memcpy(buf1.get(), v1.data(), v1.size()); + std::memcpy(buf2.get(), v2.data(), v2.size()); + batch.emplace(k1, std::vector{Slice{buf1.get(), v1.size()}}); + batch.emplace(k2, std::vector{Slice{buf2.get(), v2.size()}}); + + auto offload_result = storage_backend.BatchOffload( + batch, + [](const std::vector&, + std::vector&) { return ErrorCode::OK; }); + ASSERT_TRUE(offload_result.has_value()); + + EXPECT_TRUE(storage_backend.IsExist(k1).value()); + EXPECT_TRUE(storage_backend.IsExist(k2).value()); + + // Mark k1 removed + storage_backend.MarkRemoved(k1); + + // k1 invisible, k2 still visible + EXPECT_FALSE(storage_backend.IsExist(k1).value()); + EXPECT_TRUE(storage_backend.IsExist(k2).value()); + + // MarkRemoved is idempotent on absent key + storage_backend.MarkRemoved("nonexistent_key"); // no crash + storage_backend.MarkRemoved(k1); // already removed, idempotent +} + +TEST_F(StorageBackendTest, BucketStorageBackend_CompactReclaimsDeletedKeys) { + FileStorageConfig config; + config.storage_filepath = data_path; + BucketBackendConfig bucket_config; + bucket_config.eviction_policy = BucketEvictionPolicy::LRU; + bucket_config.disable_ssd_eviction = true; + BucketStorageBackend storage_backend(config, bucket_config); + ASSERT_TRUE(storage_backend.Init()); + + // Offload 3 keys into one bucket + std::string k1 = "compact_k1", k2 = "compact_k2", k3 = "compact_k3"; + std::string v1(1024, 'a'), v2(1024, 'b'), v3(1024, 'c'); + + std::unordered_map> batch; + auto buf1 = std::make_unique(v1.size()); + auto buf2 = std::make_unique(v2.size()); + auto buf3 = std::make_unique(v3.size()); + std::memcpy(buf1.get(), v1.data(), v1.size()); + std::memcpy(buf2.get(), v2.data(), v2.size()); + std::memcpy(buf3.get(), v3.data(), v3.size()); + batch.emplace(k1, std::vector{Slice{buf1.get(), v1.size()}}); + batch.emplace(k2, std::vector{Slice{buf2.get(), v2.size()}}); + batch.emplace(k3, std::vector{Slice{buf3.get(), v3.size()}}); + + auto offload_result = storage_backend.BatchOffload( + batch, + [](const std::vector&, + std::vector&) { return ErrorCode::OK; }); + ASSERT_TRUE(offload_result.has_value()); + int64_t old_bucket_id = offload_result.value(); + + // Mark k2 removed + storage_backend.MarkRemoved(k2); + + // Compact the bucket + ASSERT_TRUE(storage_backend.CompactBucket(old_bucket_id)); + + // k1, k3 still loadable with correct data; k2 gone + EXPECT_TRUE(storage_backend.IsExist(k1).value()); + EXPECT_TRUE(storage_backend.IsExist(k3).value()); + EXPECT_FALSE(storage_backend.IsExist(k2).value()); + + // Verify k1, k3 data integrity + auto alloc = SimpleAllocator(128 * 1024 * 1024); + void* b1 = alloc.allocate(v1.size()); + void* b3 = alloc.allocate(v3.size()); + std::unordered_map load_batch; + load_batch.emplace(k1, Slice{b1, v1.size()}); + load_batch.emplace(k3, Slice{b3, v3.size()}); + ASSERT_TRUE(storage_backend.BatchLoad(load_batch)); + EXPECT_EQ(std::string((char*)b1, v1.size()), v1); + EXPECT_EQ(std::string((char*)b3, v3.size()), v3); + + // Old bucket file should be deleted + std::string old_data_path = + data_path + "/" + std::to_string(old_bucket_id) + ".bucket"; + EXPECT_FALSE(fs::exists(old_data_path)) + << "Old bucket file should be deleted after compaction"; +} + +TEST_F(StorageBackendTest, BucketStorageBackend_MarkRemovedConcurrentLoad) { + FileStorageConfig config; + config.storage_filepath = data_path; + BucketBackendConfig bucket_config; + bucket_config.eviction_policy = BucketEvictionPolicy::LRU; + bucket_config.disable_ssd_eviction = true; + BucketStorageBackend storage_backend(config, bucket_config); + ASSERT_TRUE(storage_backend.Init()); + + std::string k1 = "conc_k1", k2 = "conc_k2"; + std::string v1(4096, 'a'), v2(4096, 'b'); + std::unordered_map> batch; + auto buf1 = std::make_unique(v1.size()); + auto buf2 = std::make_unique(v2.size()); + std::memcpy(buf1.get(), v1.data(), v1.size()); + std::memcpy(buf2.get(), v2.data(), v2.size()); + batch.emplace(k1, std::vector{Slice{buf1.get(), v1.size()}}); + batch.emplace(k2, std::vector{Slice{buf2.get(), v2.size()}}); + ASSERT_TRUE(storage_backend.BatchOffload( + batch, [](const std::vector&, + std::vector&) { + return ErrorCode::OK; + })); + + // Concurrent: load k1 in a thread while marking k2 removed. + auto alloc = SimpleAllocator(128 * 1024 * 1024); + void* b1 = alloc.allocate(v1.size()); + std::thread loader([&]() { + std::unordered_map load; + load.emplace(k1, Slice{b1, v1.size()}); + auto r = storage_backend.BatchLoad(load); + ASSERT_TRUE(r); + }); + + storage_backend.MarkRemoved(k2); + loader.join(); + + // k1 data intact, k2 gone. + EXPECT_EQ(std::string((char*)b1, v1.size()), v1); + EXPECT_FALSE(storage_backend.IsExist(k2).value()); + EXPECT_TRUE(storage_backend.IsExist(k1).value()); +} + +TEST_F(StorageBackendTest, + BucketStorageBackend_DisableEvictionNoopUnderPressure) { + FileStorageConfig config; + config.storage_filepath = data_path; + // Set global size limit smaller than a single bucket's reservation so + // IsEnableOffloading's quota check rejects the offload. + config.total_size_limit = 512; + BucketBackendConfig bucket_config; + bucket_config.eviction_policy = BucketEvictionPolicy::LRU; + bucket_config.disable_ssd_eviction = true; + // bucket_size_limit (default 256MB) >> total_size_limit (512), so the + // quota check in IsEnableOffloading rejects without eviction. + BucketStorageBackend storage_backend(config, bucket_config); + ASSERT_TRUE(storage_backend.Init()); + + // disable_ssd_eviction=true means PrepareEviction is a no-op, so under + // space pressure no bucket is deleted. IsEnableOffloading rejects via + // the quota check instead. + std::string k = "pressure_k"; + std::string v(2048, 'z'); + auto buf = std::make_unique(v.size()); + std::memcpy(buf.get(), v.data(), v.size()); + std::unordered_map> batch; + batch.emplace(k, std::vector{Slice{buf.get(), v.size()}}); + + auto result = storage_backend.BatchOffload( + batch, [](const std::vector&, + std::vector&) { + return ErrorCode::OK; + }); + // Offload should be rejected by quota (no eviction to reclaim space). + EXPECT_FALSE(result.has_value()); +} + +//----------------------------------------------------------------------------- +// Cross-bucket merge compaction tests +//----------------------------------------------------------------------------- + +TEST_F(StorageBackendTest, BucketStorageBackend_CrossBucketMergeCompaction) { + FileStorageConfig config; + config.storage_filepath = data_path; + BucketBackendConfig bucket_config; + bucket_config.eviction_policy = BucketEvictionPolicy::LRU; + bucket_config.disable_ssd_eviction = true; + // Set keys_limit=2 so 2 keys fill a bucket. We'll create 2 buckets with + // 2 keys each (4 keys total), remove 1 key from each (2 tombstones), + // then CompactBuckets should merge the 2 remaining live keys from each + // bucket into 1 new bucket. + bucket_config.bucket_keys_limit = 2; + BucketStorageBackend storage_backend(config, bucket_config); + ASSERT_TRUE(storage_backend.Init()); + + // Offload 4 keys into 2 buckets (bucket_keys_limit=2). + // Batch 1: k1, k2 -> bucket A + // Batch 2: k3, k4 -> bucket B + auto offload_batch = [&](const std::vector>& kvs) { + std::unordered_map> batch; + std::vector> bufs; + for (const auto& [k, v] : kvs) { + auto buf = std::make_unique(v.size()); + std::memcpy(buf.get(), v.data(), v.size()); + bufs.push_back(std::move(buf)); + batch.emplace(k, std::vector{ + Slice{bufs.back().get(), v.size()}}); + } + return storage_backend.BatchOffload( + batch, + [](const std::vector&, + std::vector&) { + return ErrorCode::OK; + }); + }; + + std::string k1 = "merge_k1", k2 = "merge_k2"; + std::string k3 = "merge_k3", k4 = "merge_k4"; + std::string v1(1024, 'a'), v2(1024, 'b'); + std::string v3(1024, 'c'), v4(1024, 'd'); + + auto offload_a = offload_batch({{k1, v1}, {k2, v2}}); + ASSERT_TRUE(offload_a.has_value()); + int64_t bucket_a = offload_a.value(); + auto offload_b = offload_batch({{k3, v3}, {k4, v4}}); + ASSERT_TRUE(offload_b.has_value()); + int64_t bucket_b = offload_b.value(); + + // Remove k2 from bucket A and k4 from bucket B (tombstones). + storage_backend.MarkRemoved(k2); + storage_backend.MarkRemoved(k4); + + // CompactBuckets should merge live keys k1, k3 into a new bucket. + // bucket_keys_limit=2, so k1+k3 fills exactly one bucket. + ASSERT_TRUE(storage_backend.CompactBuckets({bucket_a, bucket_b})); + + // k1 and k3 must still be readable with correct data. + EXPECT_TRUE(storage_backend.IsExist(k1).value()); + EXPECT_TRUE(storage_backend.IsExist(k3).value()); + // k2 and k4 must be gone. + EXPECT_FALSE(storage_backend.IsExist(k2).value()); + EXPECT_FALSE(storage_backend.IsExist(k4).value()); + + // Verify data integrity. + auto alloc = SimpleAllocator(128 * 1024 * 1024); + void* b1 = alloc.allocate(v1.size()); + void* b3 = alloc.allocate(v3.size()); + std::unordered_map load_batch; + load_batch.emplace(k1, Slice{b1, v1.size()}); + load_batch.emplace(k3, Slice{b3, v3.size()}); + ASSERT_TRUE(storage_backend.BatchLoad(load_batch)); + EXPECT_EQ(std::string((char*)b1, v1.size()), v1); + EXPECT_EQ(std::string((char*)b3, v3.size()), v3); + + // Old bucket files should be deleted (both A and B fully migrated). + std::string path_a = data_path + "/" + std::to_string(bucket_a) + ".bucket"; + std::string path_b = data_path + "/" + std::to_string(bucket_b) + ".bucket"; + EXPECT_FALSE(fs::exists(path_a)) + << "Old bucket A file should be deleted after merge"; + EXPECT_FALSE(fs::exists(path_b)) + << "Old bucket B file should be deleted after merge"; +} + +TEST_F(StorageBackendTest, BucketStorageBackend_MergeDeferredWhenNotFull) { + FileStorageConfig config; + config.storage_filepath = data_path; + BucketBackendConfig bucket_config; + bucket_config.eviction_policy = BucketEvictionPolicy::LRU; + bucket_config.disable_ssd_eviction = true; + // keys_limit=2: need 2 live keys to fill a bucket. + bucket_config.bucket_keys_limit = 2; + BucketStorageBackend storage_backend(config, bucket_config); + ASSERT_TRUE(storage_backend.Init()); + + // Offload 2 keys into 1 bucket. + std::unordered_map> batch; + auto buf1 = std::make_unique(1024); + auto buf2 = std::make_unique(1024); + std::memset(buf1.get(), 'x', 1024); + std::memset(buf2.get(), 'y', 1024); + batch.emplace("defer_k1", std::vector{Slice{buf1.get(), 1024}}); + batch.emplace("defer_k2", std::vector{Slice{buf2.get(), 1024}}); + auto offload_result = storage_backend.BatchOffload( + batch, + [](const std::vector&, + std::vector&) { return ErrorCode::OK; }); + ASSERT_TRUE(offload_result.has_value()); + int64_t bucket_id = offload_result.value(); + + // Remove k2 — only 1 live key (k1) remains. Not enough for keys_limit=2. + storage_backend.MarkRemoved("defer_k2"); + + // CompactBuckets without space_pressure should defer (return true, no + // compaction). The old bucket should still exist. + ASSERT_TRUE(storage_backend.CompactBuckets({bucket_id}, false)); + EXPECT_TRUE(storage_backend.IsExist("defer_k1").value()); + // Old bucket file should still exist (not compacted). + std::string path = data_path + "/" + std::to_string(bucket_id) + ".bucket"; + EXPECT_TRUE(fs::exists(path)) + << "Bucket should not be compacted when live keys don't fill a bucket"; + + // With space_pressure=true, compaction should proceed even if not full. + ASSERT_TRUE(storage_backend.CompactBuckets({bucket_id}, true)); + EXPECT_TRUE(storage_backend.IsExist("defer_k1").value()); + EXPECT_FALSE(fs::exists(path)) + << "Bucket should be compacted under space pressure"; +} + + TEST_F(StorageBackendTest, StoreObjectReturnsEvictedKeys) { std::string test_dir = data_path + "/evict_return_test"; std::filesystem::create_directories(test_dir); diff --git a/mooncake-store/tests/vchunk_allocation_strategy_test.cpp b/mooncake-store/tests/vchunk_allocation_strategy_test.cpp new file mode 100644 index 0000000000..e5fb1fd82e --- /dev/null +++ b/mooncake-store/tests/vchunk_allocation_strategy_test.cpp @@ -0,0 +1,118 @@ +#include "vchunk_allocation_strategy.h" + +#include + +#include +#include +#include +#include +#include + +#include "allocation_strategy.h" +#include "vchunk_test_allocator.h" + +namespace mooncake { +namespace { + +using test::VChunkTestAllocator; + +TEST(VChunkAllocationStrategyTest, AllocatesTransposeRowsAndRollsBack) { + AllocatorManager manager; + std::vector> allocators; + for (size_t i = 0; i < 3; ++i) { + auto allocator = std::make_shared( + "segment-" + std::to_string(i), 0x100000000ULL + i * 0x100000, + 64U * 1024U); + manager.addAllocator(allocator->getSegmentName(), allocator); + allocators.push_back(std::move(allocator)); + } + + { + auto result = AllocateVChunk(manager, 9U * 4096U, + VCSliceSizeLevel::k4K); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result->row_size, 3U); + ASSERT_EQ(result->allocations.size(), 9U); + for (size_t row = 0; row < 3; ++row) { + std::unordered_set segments; + for (size_t column = 0; column < 3; ++column) { + const auto& allocation = + result->allocations[row * 3 + column]; + EXPECT_TRUE(segments.insert(allocation.segment_name).second); + EXPECT_EQ(allocation.slice_index, row * 3 + column); + EXPECT_EQ(allocation.logical_length, 4096U); + EXPECT_EQ(allocation.allocated_length, 4096U); + EXPECT_NE(allocation.buffer, nullptr); + } + } + for (const auto& allocator : allocators) { + EXPECT_EQ(allocator->size(), 3U * 4096U); + } + } + for (const auto& allocator : allocators) { + EXPECT_EQ(allocator->size(), 0U); + } +} + +TEST(VChunkAllocationStrategyTest, TracksShortFinalLogicalSlice) { + AllocatorManager manager; + auto allocator = std::make_shared( + "segment-a", 0x200000000ULL, 64U * 1024U); + manager.addAllocator("segment-a", allocator); + + auto result = + AllocateVChunk(manager, 10U * 1024U, VCSliceSizeLevel::k4K); + ASSERT_TRUE(result.has_value()); + ASSERT_EQ(result->allocations.size(), 3U); + EXPECT_EQ(result->allocations.back().logical_length, 2048U); + EXPECT_EQ(result->allocations.back().allocated_length, 4096U); +} + +TEST(VChunkAllocationStrategyTest, HonorsExcludedSegments) { + AllocatorManager manager; + auto first = std::make_shared( + "segment-a", 0x300000000ULL, 64U * 1024U); + auto second = std::make_shared( + "segment-b", 0x400000000ULL, 64U * 1024U); + manager.addAllocator("segment-a", first); + manager.addAllocator("segment-b", second); + + auto result = AllocateVChunk(manager, 8192, VCSliceSizeLevel::k4K, + std::set{"segment-a"}); + ASSERT_TRUE(result.has_value()); + for (const auto& allocation : result->allocations) { + EXPECT_EQ(allocation.segment_name, "segment-b"); + } + EXPECT_EQ(first->size(), 0U); +} + +TEST(VChunkAllocationStrategyTest, PartialFailureRollsBackAllBuffers) { + AllocatorManager manager; + auto first = std::make_shared( + "segment-a", 0x500000000ULL, 4096); + auto second = std::make_shared( + "segment-b", 0x600000000ULL, 4096); + manager.addAllocator("segment-a", first); + manager.addAllocator("segment-b", second); + + auto result = + AllocateVChunk(manager, 3U * 4096U, VCSliceSizeLevel::k4K); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error(), ErrorCode::NO_AVAILABLE_HANDLE); + EXPECT_EQ(first->size(), 0U); + EXPECT_EQ(second->size(), 0U); +} + +TEST(VChunkAllocationStrategyTest, RejectsInvalidAndEmptyInputs) { + AllocatorManager manager; + auto empty = AllocateVChunk(manager, 4096, VCSliceSizeLevel::k4K); + EXPECT_FALSE(empty.has_value()); + EXPECT_EQ(empty.error(), ErrorCode::NO_AVAILABLE_HANDLE); + + auto zero = AllocateVChunk(manager, 0, VCSliceSizeLevel::k4K); + EXPECT_FALSE(zero.has_value()); + EXPECT_EQ(zero.error(), ErrorCode::INVALID_PARAMS); +} + +} // namespace +} // namespace mooncake diff --git a/mooncake-store/tests/vchunk_client_test.cpp b/mooncake-store/tests/vchunk_client_test.cpp new file mode 100644 index 0000000000..1b52e200c4 --- /dev/null +++ b/mooncake-store/tests/vchunk_client_test.cpp @@ -0,0 +1,399 @@ +#include "vchunk_client.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "master_config.h" +#include "master_service.h" + +namespace mooncake { +namespace { + +Segment MakeSegment(const std::string& name, uintptr_t base) { + Segment segment; + segment.name = name; + segment.base = base; + segment.size = 32U * 1024U * 1024U; + segment.protocol = "tcp"; + segment.te_endpoint = name; + return segment; +} + +class MemoryDataPlane final : public VChunkDataPlane { + public: + ErrorCode Write(const VChunkMetadataRecord& record, const void* source, + size_t length, + std::chrono::steady_clock::time_point deadline) override { + ++write_attempts; + if (write_failures_remaining > 0) { + --write_failures_remaining; + return ErrorCode::TRANSFER_FAIL; + } + if (fail_write || std::chrono::steady_clock::now() >= deadline) { + return fail_write ? ErrorCode::TRANSFER_FAIL + : ErrorCode::RPC_TIMEOUT; + } + if (!source || record.total_size != length) { + return ErrorCode::INVALID_PARAMS; + } + std::vector> slices; + size_t offset = 0; + for (const auto& slice : record.slices) { + if (slice.slice_index != slices.size() || + slice.logical_length > length - offset) { + return ErrorCode::INVALID_PARAMS; + } + const auto* begin = static_cast(source) + offset; + slices.emplace_back(begin, begin + slice.logical_length); + offset += slice.logical_length; + } + if (offset != length) { + return ErrorCode::INVALID_PARAMS; + } + std::lock_guard guard(mutex); + objects[record.vchunk_id] = std::move(slices); + return ErrorCode::OK; + } + + ErrorCode Read(const VChunkMetadataRecord& record, void* destination, + size_t length, + std::chrono::steady_clock::time_point deadline) override { + ++read_attempts; + if (read_failures_remaining > 0) { + --read_failures_remaining; + return ErrorCode::TRANSFER_FAIL; + } + if (fail_read || std::chrono::steady_clock::now() >= deadline) { + return fail_read ? ErrorCode::TRANSFER_FAIL + : ErrorCode::RPC_TIMEOUT; + } + if (block_reads) { + std::unique_lock guard(sync_mutex); + read_entered = true; + sync_cv.notify_all(); + sync_cv.wait(guard, [&] { return release_read; }); + } + std::lock_guard guard(mutex); + const auto it = objects.find(record.vchunk_id); + if (it == objects.end()) { + return ErrorCode::OBJECT_NOT_FOUND; + } + size_t offset = 0; + for (size_t i = 0; i < it->second.size(); ++i) { + if (record.slices[i].slice_index != i || + it->second[i].size() > length - offset) { + return ErrorCode::TRANSFER_FAIL; + } + std::memcpy(static_cast(destination) + offset, + it->second[i].data(), it->second[i].size()); + offset += it->second[i].size(); + } + return offset == length ? ErrorCode::OK : ErrorCode::TRANSFER_FAIL; + } + + bool fail_write{false}; + bool fail_read{false}; + int write_failures_remaining{0}; + int read_failures_remaining{0}; + int write_attempts{0}; + int read_attempts{0}; + bool block_reads{false}; + bool read_entered{false}; + bool release_read{false}; + std::mutex sync_mutex; + std::condition_variable sync_cv; + std::mutex mutex; + std::unordered_map>> objects; +}; + +class LegacySpy final : public VChunkLegacyPath { + public: + ErrorCode Put(const TenantId&, const std::string&, const void*, + size_t) override { + ++puts; + return ErrorCode::OK; + } + ErrorCode Get(const TenantId&, const std::string&, void*, size_t) override { + ++gets; + return ErrorCode::OK; + } + ErrorCode Remove(const TenantId&, const std::string&) override { + ++removes; + return ErrorCode::OK; + } + int puts{0}; + int gets{0}; + int removes{0}; +}; + +class DelayingControlPlane final : public VChunkControlPlane { + public: + DelayingControlPlane(VChunkControlPlane& delegate, + std::chrono::milliseconds delay) + : delegate_(delegate), delay_(delay) {} + + tl::expected PutStart( + const TenantId& tenant_id, const std::string& key, uint64_t total_size, + int64_t now_ms) override { + std::this_thread::sleep_for(delay_); + return delegate_.PutStart(tenant_id, key, total_size, now_ms); + } + + ErrorCode PutEnd(const TenantId& tenant_id, const std::string& key, + const std::string& vchunk_id, int64_t now_ms) override { + return delegate_.PutEnd(tenant_id, key, vchunk_id, now_ms); + } + + ErrorCode PutRevoke(const TenantId& tenant_id, const std::string& key, + const std::string& vchunk_id) override { + return delegate_.PutRevoke(tenant_id, key, vchunk_id); + } + + tl::expected Get( + const TenantId& tenant_id, const std::string& key) override { + std::this_thread::sleep_for(delay_); + return delegate_.Get(tenant_id, key); + } + + ErrorCode Remove(const TenantId& tenant_id, const std::string& key, + int64_t now_ms) override { + return delegate_.Remove(tenant_id, key, now_ms); + } + + private: + VChunkControlPlane& delegate_; + std::chrono::milliseconds delay_; +}; + +struct ClientFixture : testing::Test { + ClientFixture() : service(MakeConfig()) { + const auto client_id = generate_uuid(); + EXPECT_TRUE(service.MountSegment( + MakeSegment("vchunk-a", 0xB00000000ULL), + client_id) + .has_value()); + EXPECT_TRUE(service.MountSegment( + MakeSegment("vchunk-b", 0xC00000000ULL), + client_id) + .has_value()); + } + + static MasterServiceConfig MakeConfig() { + MasterServiceConfig config; + config.memory_allocator = BufferAllocatorType::OFFSET; + config.vchunk_config.enabled = true; + return config; + } + + int64_t now{100}; + MasterService service; + MemoryDataPlane data; + LegacySpy legacy; +}; + +TEST_F(ClientFixture, PutGetRemoveRoundTripForPiercingSizes) { + const std::array sizes{4096, 64U * 1024U, 256U * 1024U, + 1024U * 1024U, 4U * 1024U * 1024U, + 1024U * 1024U + 17U}; + VChunkClient client(true, service, data, legacy, + std::chrono::seconds(1), [this] { return ++now; }); + std::mt19937 random(7); + for (const auto size : sizes) { + std::vector source(size); + std::generate(source.begin(), source.end(), [&] { return random(); }); + std::vector destination(size, 0); + const auto key = "key-" + std::to_string(size); + ASSERT_EQ(client.Put(TenantId("tenant"), key, source.data(), size), + ErrorCode::OK); + ASSERT_EQ(client.Get(TenantId("tenant"), key, destination.data(), size), + ErrorCode::OK); + EXPECT_EQ(destination, source); + EXPECT_EQ(client.Remove(TenantId("tenant"), key), ErrorCode::OK); + EXPECT_EQ(client.Remove(TenantId("tenant"), key), ErrorCode::OK); + EXPECT_EQ(client.Get(TenantId("tenant"), key, destination.data(), size), + ErrorCode::OBJECT_NOT_FOUND); + } +} + +TEST_F(ClientFixture, FailedWriteRevokesCreatingObject) { + VChunkClient client(true, service, data, legacy, + std::chrono::seconds(1), [this] { return ++now; }); + std::vector source(8192, 1); + data.fail_write = true; + EXPECT_EQ(client.Put(TenantId("tenant"), "key", source.data(), + source.size()), + ErrorCode::TRANSFER_FAIL); + EXPECT_EQ(service.GetVChunk(TenantId("tenant"), "key").error(), + ErrorCode::OBJECT_NOT_FOUND); +} + +TEST_F(ClientFixture, FailedReadDoesNotReturnPartialSuccess) { + VChunkClient client(true, service, data, legacy, + std::chrono::seconds(1), [this] { return ++now; }); + std::vector source(8192, 3); + ASSERT_EQ(client.Put(TenantId("tenant"), "key", source.data(), + source.size()), + ErrorCode::OK); + std::vector destination(source.size(), 0); + data.fail_read = true; + EXPECT_EQ(client.Get(TenantId("tenant"), "key", destination.data(), + destination.size()), + ErrorCode::TRANSFER_FAIL); +} + +TEST_F(ClientFixture, DisabledVChunkUsesOnlyLegacyPath) { + VChunkClient client(false, service, data, legacy, + std::chrono::seconds(1), [this] { return ++now; }); + uint8_t value = 1; + EXPECT_EQ(client.Put(TenantId("tenant"), "key", &value, 1), ErrorCode::OK); + EXPECT_EQ(client.Get(TenantId("tenant"), "key", &value, 1), ErrorCode::OK); + EXPECT_EQ(client.Remove(TenantId("tenant"), "key"), ErrorCode::OK); + EXPECT_EQ(legacy.puts, 1); + EXPECT_EQ(legacy.gets, 1); + EXPECT_EQ(legacy.removes, 1); + EXPECT_EQ(service.GetVChunk(TenantId("tenant"), "key").error(), + ErrorCode::OBJECT_NOT_FOUND); +} + +TEST_F(ClientFixture, InflightGetCompletesWhileRemoveBlocksNewReads) { + VChunkClient client(true, service, data, legacy, + std::chrono::seconds(1), [this] { return ++now; }); + std::vector source(8192, 9); + ASSERT_EQ(client.Put(TenantId("tenant"), "key", source.data(), + source.size()), + ErrorCode::OK); + data.block_reads = true; + std::vector destination(source.size(), 0); + ErrorCode read_result = ErrorCode::INTERNAL_ERROR; + std::thread reader([&] { + read_result = client.Get(TenantId("tenant"), "key", destination.data(), + destination.size()); + }); + { + std::unique_lock guard(data.sync_mutex); + data.sync_cv.wait(guard, [&] { return data.read_entered; }); + } + + EXPECT_EQ(client.Remove(TenantId("tenant"), "key"), ErrorCode::OK); + std::vector second(source.size(), 0); + EXPECT_EQ(client.Get(TenantId("tenant"), "key", second.data(), + second.size()), + ErrorCode::OBJECT_NOT_FOUND); + { + std::lock_guard guard(data.sync_mutex); + data.release_read = true; + } + data.sync_cv.notify_all(); + reader.join(); + EXPECT_EQ(read_result, ErrorCode::OK); + EXPECT_EQ(destination, source); +} + +TEST_F(ClientFixture, BatchKeepsPerKeyFailuresIndependent) { + VChunkClient client(true, service, data, legacy, + std::chrono::seconds(1), [this] { return ++now; }); + std::vector first(4096, 1); + std::vector third(4096, 3); + const std::vector puts{ + {"first", first.data(), first.size()}, {"bad", nullptr, 4096}, + {"third", third.data(), third.size()}}; + const auto put_results = client.BatchPut(TenantId("tenant"), puts); + ASSERT_EQ(put_results.size(), 3U); + EXPECT_EQ(put_results[0], ErrorCode::OK); + EXPECT_EQ(put_results[1], ErrorCode::INVALID_PARAMS); + EXPECT_EQ(put_results[2], ErrorCode::OK); + + std::vector first_out(first.size()); + std::vector missing(first.size()); + std::vector third_out(third.size()); + const std::vector gets{ + {"first", first_out.data(), first_out.size()}, + {"missing", missing.data(), missing.size()}, + {"third", third_out.data(), third_out.size()}}; + const auto get_results = client.BatchGet(TenantId("tenant"), gets); + EXPECT_EQ(get_results[0], ErrorCode::OK); + EXPECT_EQ(get_results[1], ErrorCode::OBJECT_NOT_FOUND); + EXPECT_EQ(get_results[2], ErrorCode::OK); + EXPECT_EQ(first_out, first); + EXPECT_EQ(third_out, third); +} + +TEST_F(ClientFixture, RetriesRetryableTransfersWithinConfiguredLimit) { + VChunkClient client(true, service, data, legacy, + std::chrono::seconds(1), [this] { return ++now; }, 2); + std::vector source(4096, 5); + data.write_failures_remaining = 1; + EXPECT_EQ(client.Put(TenantId("tenant"), "key", source.data(), + source.size()), + ErrorCode::OK); + EXPECT_EQ(data.write_attempts, 2); + const auto metrics = client.MetricsSnapshot(); + EXPECT_EQ(metrics.retries, 1U); + EXPECT_EQ(metrics.requests[static_cast(VChunkOperation::PUT)], 1U); + EXPECT_EQ(metrics.successes[static_cast(VChunkOperation::PUT)], 1U); +} + +TEST_F(ClientFixture, CircuitBreakerStopsOnlyNewVChunkCreation) { + VChunkClient client(true, service, data, legacy, + std::chrono::seconds(1), [this] { return ++now; }, 0, + 1); + std::vector source(4096, 5); + ASSERT_EQ(client.Put(TenantId("tenant"), "active", source.data(), + source.size()), + ErrorCode::OK); + data.fail_write = true; + EXPECT_EQ(client.Put(TenantId("tenant"), "failed", source.data(), + source.size()), + ErrorCode::TRANSFER_FAIL); + data.fail_write = false; + EXPECT_EQ(client.Put(TenantId("tenant"), "blocked", source.data(), + source.size()), + ErrorCode::NO_AVAILABLE_HANDLE); + EXPECT_EQ(data.write_attempts, 2); + std::vector output(source.size()); + EXPECT_EQ(client.Get(TenantId("tenant"), "active", output.data(), + output.size()), + ErrorCode::OK); + EXPECT_EQ(output, source); + EXPECT_EQ(client.Remove(TenantId("tenant"), "active"), ErrorCode::OK); +} + +TEST_F(ClientFixture, ControlPlaneLatencyConsumesOperationTimeout) { + LocalVChunkControlPlane local(service); + DelayingControlPlane delayed(local, std::chrono::milliseconds(20)); + VChunkClient timed_client(true, delayed, data, legacy, + std::chrono::milliseconds(1), + [this] { return ++now; }); + std::vector source(4096, 7); + EXPECT_EQ(timed_client.Put(TenantId("tenant"), "slow-put", source.data(), + source.size()), + ErrorCode::RPC_TIMEOUT); + EXPECT_EQ(service.GetVChunk(TenantId("tenant"), "slow-put").error(), + ErrorCode::OBJECT_NOT_FOUND); + + VChunkClient setup_client(true, service, data, legacy, + std::chrono::seconds(1), + [this] { return ++now; }); + ASSERT_EQ(setup_client.Put(TenantId("tenant"), "slow-get", source.data(), + source.size()), + ErrorCode::OK); + std::vector destination(source.size()); + EXPECT_EQ(timed_client.Get(TenantId("tenant"), "slow-get", + destination.data(), destination.size()), + ErrorCode::RPC_TIMEOUT); + EXPECT_EQ(data.read_attempts, 0); +} + +} // namespace +} // namespace mooncake diff --git a/mooncake-store/tests/vchunk_config_test.cpp b/mooncake-store/tests/vchunk_config_test.cpp new file mode 100644 index 0000000000..88e0e7960d --- /dev/null +++ b/mooncake-store/tests/vchunk_config_test.cpp @@ -0,0 +1,63 @@ +#include "vchunk_config.h" + +#include + +namespace mooncake { +namespace { + +TEST(VChunkConfigTest, DefaultsAreSafeForProductionOptIn) { + VChunkConfig config; + EXPECT_FALSE(config.enabled); + EXPECT_EQ(config.Validate(), ErrorCode::OK); + EXPECT_EQ(config.max_slice_count, 4096U); + EXPECT_EQ(config.max_metadata_bytes, 1024U * 1024U); + EXPECT_EQ(config.max_creating_objects, 1024U); + EXPECT_EQ(config.reaper_interval_ms, 1000U); + EXPECT_EQ(config.reaper_max_scan, 128U); +} + +TEST(VChunkConfigTest, RejectsInvalidLimits) { + VChunkConfig config; + config.max_slice_count = 0; + EXPECT_EQ(config.Validate(), ErrorCode::INVALID_PARAMS); + + config = VChunkConfig{}; + config.max_metadata_bytes = 0; + EXPECT_EQ(config.Validate(), ErrorCode::INVALID_PARAMS); + + config = VChunkConfig{}; + config.creating_timeout_ms = 0; + EXPECT_EQ(config.Validate(), ErrorCode::INVALID_PARAMS); + + config = VChunkConfig{}; + config.max_creating_objects = 0; + EXPECT_EQ(config.Validate(), ErrorCode::INVALID_PARAMS); + + config = VChunkConfig{}; + config.reaper_interval_ms = 0; + EXPECT_EQ(config.Validate(), ErrorCode::INVALID_PARAMS); +} + +TEST(VChunkConfigTest, SelectsMemorySliceBoundaries) { + EXPECT_EQ(SelectVChunkSliceSize(0, false), VCSliceSizeLevel::k4K); + EXPECT_EQ(SelectVChunkSliceSize(64U * 1024U - 1, false), + VCSliceSizeLevel::k4K); + EXPECT_EQ(SelectVChunkSliceSize(64U * 1024U, false), + VCSliceSizeLevel::k64K); + EXPECT_EQ(SelectVChunkSliceSize(256U * 1024U - 1, false), + VCSliceSizeLevel::k64K); + EXPECT_EQ(SelectVChunkSliceSize(256U * 1024U, false), + VCSliceSizeLevel::k256K); + EXPECT_EQ(SelectVChunkSliceSize(1024U * 1024U - 1, false), + VCSliceSizeLevel::k256K); + EXPECT_EQ(SelectVChunkSliceSize(1024U * 1024U, false), + VCSliceSizeLevel::k1M); +} + +TEST(VChunkConfigTest, SsdAlwaysUsesFourKiB) { + EXPECT_EQ(SelectVChunkSliceSize(8U * 1024U * 1024U, true), + VCSliceSizeLevel::k4K); +} + +} // namespace +} // namespace mooncake diff --git a/mooncake-store/tests/vchunk_master_manager_test.cpp b/mooncake-store/tests/vchunk_master_manager_test.cpp new file mode 100644 index 0000000000..a85694879e --- /dev/null +++ b/mooncake-store/tests/vchunk_master_manager_test.cpp @@ -0,0 +1,216 @@ +#include "vchunk_master_manager.h" + +#include + +#include +#include +#include +#include +#include + +#include "allocation_strategy.h" +#include "vchunk_test_allocator.h" + +namespace mooncake { +namespace { + +using test::VChunkTestAllocator; + +struct ManagerFixture { + AllocatorManager allocators; + std::shared_ptr first = + std::make_shared("segment-a", 0x700000000ULL, + 1024U * 1024U); + std::shared_ptr second = + std::make_shared("segment-b", 0x800000000ULL, + 1024U * 1024U); + + ManagerFixture() { + allocators.addAllocator("segment-a", first); + allocators.addAllocator("segment-b", second); + } +}; + +VChunkConfig EnabledConfig() { + VChunkConfig config; + config.enabled = true; + return config; +} + +TEST(VChunkMasterManagerTest, RunsPutGetRemoveLifecycle) { + ManagerFixture fixture; + VChunkMasterManager manager(EnabledConfig()); + const TenantId tenant("tenant-a"); + + auto created = manager.PutStart(fixture.allocators, tenant, "key", 10U * 1024U, + false, 100); + ASSERT_TRUE(created.has_value()); + EXPECT_EQ(created->status, VChunkStatus::CREATING); + EXPECT_EQ(created->slice_count, 3U); + EXPECT_EQ(created->row_size, 2U); + EXPECT_FALSE(manager.Get(tenant, "key").has_value()); + + EXPECT_EQ(manager.PutEnd(tenant, "key", "wrong-id", 200), + ErrorCode::INVALID_VERSION); + EXPECT_EQ(manager.PutEnd(tenant, "key", created->vchunk_id, 200), + ErrorCode::OK); + EXPECT_EQ(manager.PutEnd(tenant, "key", created->vchunk_id, 200), + ErrorCode::OK); + + auto active = manager.Get(tenant, "key"); + ASSERT_TRUE(active.has_value()); + EXPECT_EQ(active->status, VChunkStatus::ACTIVE); + for (const auto& slice : active->slices) { + EXPECT_EQ(slice.status, VCSliceStatus::COMPLETED); + } + + EXPECT_EQ(manager.Remove(tenant, "key", 300), ErrorCode::OK); + EXPECT_EQ(manager.Remove(tenant, "key", 301), ErrorCode::OK); + EXPECT_EQ(manager.SizeForTesting(), 0U); + EXPECT_EQ(fixture.first->size(), 0U); + EXPECT_EQ(fixture.second->size(), 0U); +} + +TEST(VChunkMasterManagerTest, RevokeIsIdempotentAndReleasesBuffers) { + ManagerFixture fixture; + VChunkMasterManager manager(EnabledConfig()); + const TenantId tenant("tenant-a"); + auto created = manager.PutStart(fixture.allocators, tenant, "key", 8192, + false, 100); + ASSERT_TRUE(created.has_value()); + EXPECT_GT(fixture.first->size() + fixture.second->size(), 0U); + + EXPECT_EQ(manager.PutRevoke(tenant, "key", created->vchunk_id), + ErrorCode::OK); + EXPECT_EQ(manager.PutRevoke(tenant, "key", created->vchunk_id), + ErrorCode::OK); + EXPECT_EQ(fixture.first->size() + fixture.second->size(), 0U); +} + +TEST(VChunkMasterManagerTest, ConcurrentPutStartPublishesOneObject) { + ManagerFixture fixture; + VChunkMasterManager manager(EnabledConfig()); + const TenantId tenant("tenant-a"); + std::atomic success{0}; + std::atomic already_exists{0}; + std::vector threads; + for (int i = 0; i < 8; ++i) { + threads.emplace_back([&] { + auto result = manager.PutStart(fixture.allocators, tenant, "key", + 8192, false, 100); + if (result.has_value()) { + ++success; + } else if (result.error() == ErrorCode::OBJECT_ALREADY_EXISTS) { + ++already_exists; + } + }); + } + for (auto& thread : threads) { + thread.join(); + } + EXPECT_EQ(success.load(), 1); + EXPECT_EQ(already_exists.load(), 7); + EXPECT_EQ(manager.SizeForTesting(), 1U); +} + +TEST(VChunkMasterManagerTest, IsolatesSameKeyAcrossTenants) { + ManagerFixture fixture; + VChunkMasterManager manager(EnabledConfig()); + auto first = manager.PutStart(fixture.allocators, TenantId("tenant-a"), + "key", 4096, false, 100); + auto second = manager.PutStart(fixture.allocators, TenantId("tenant-b"), + "key", 4096, false, 100); + ASSERT_TRUE(first.has_value()); + ASSERT_TRUE(second.has_value()); + EXPECT_NE(first->vchunk_id, second->vchunk_id); + EXPECT_EQ(manager.SizeForTesting(), 2U); +} + +TEST(VChunkMasterManagerTest, GetAndRemoveAreSerializedSafely) { + ManagerFixture fixture; + VChunkMasterManager manager(EnabledConfig()); + const TenantId tenant("tenant-a"); + auto created = manager.PutStart(fixture.allocators, tenant, "key", 8192, + false, 100); + ASSERT_TRUE(created.has_value()); + ASSERT_EQ(manager.PutEnd(tenant, "key", created->vchunk_id, 200), + ErrorCode::OK); + + std::atomic stop{false}; + std::atomic valid_reads{0}; + std::thread reader([&] { + while (!stop.load()) { + auto result = manager.Get(tenant, "key"); + if (result.has_value()) { + EXPECT_EQ(result->status, VChunkStatus::ACTIVE); + ++valid_reads; + } else { + EXPECT_EQ(result.error(), ErrorCode::OBJECT_NOT_FOUND); + } + } + }); + EXPECT_EQ(manager.Remove(tenant, "key", 300), ErrorCode::OK); + stop.store(true); + reader.join(); + + EXPECT_EQ(manager.SizeForTesting(), 0U); + EXPECT_EQ(fixture.first->size() + fixture.second->size(), 0U); +} + +TEST(VChunkMasterManagerTest, ReadLeaseDefersBufferReleaseAfterRemove) { + ManagerFixture fixture; + VChunkMasterManager manager(EnabledConfig()); + const TenantId tenant("tenant-a"); + auto created = manager.PutStart(fixture.allocators, tenant, "key", 8192, + false, 100); + ASSERT_TRUE(created.has_value()); + ASSERT_EQ(manager.PutEnd(tenant, "key", created->vchunk_id, 200), + ErrorCode::OK); + + const auto allocated = fixture.first->size() + fixture.second->size(); + ASSERT_GT(allocated, 0U); + { + auto read = manager.AcquireRead(tenant, "key"); + ASSERT_TRUE(read.has_value()); + EXPECT_EQ(manager.Remove(tenant, "key", 300), ErrorCode::OK); + EXPECT_FALSE(manager.Get(tenant, "key").has_value()); + EXPECT_EQ(fixture.first->size() + fixture.second->size(), allocated); + } + EXPECT_EQ(fixture.first->size() + fixture.second->size(), 0U); +} + +TEST(VChunkMasterManagerTest, DisabledConfigurationRejectsCreation) { + ManagerFixture fixture; + VChunkMasterManager manager(VChunkConfig{}); + auto result = manager.PutStart(fixture.allocators, TenantId::Default(), + "key", 4096, false, 100); + EXPECT_FALSE(result.has_value()); + EXPECT_EQ(result.error(), ErrorCode::INVALID_PARAMS); +} + +TEST(VChunkMasterManagerTest, PiercingVersionRejectsSsdSegments) { + ManagerFixture fixture; + VChunkMasterManager manager(EnabledConfig()); + auto result = manager.PutStart(fixture.allocators, TenantId::Default(), + "key", 4096, true, 100); + EXPECT_FALSE(result.has_value()); + EXPECT_EQ(result.error(), ErrorCode::INVALID_PARAMS); + EXPECT_EQ(fixture.first->size() + fixture.second->size(), 0U); +} + +TEST(VChunkMasterManagerTest, CreatingBacklogTripsAllocationCircuitBreaker) { + ManagerFixture fixture; + auto config = EnabledConfig(); + config.max_creating_objects = 1; + VChunkMasterManager manager(config); + ASSERT_TRUE(manager.PutStart(fixture.allocators, TenantId("tenant"), + "first", 4096, false, 100) + .has_value()); + auto blocked = manager.PutStart(fixture.allocators, TenantId("tenant"), + "second", 4096, false, 101); + ASSERT_FALSE(blocked.has_value()); + EXPECT_EQ(blocked.error(), ErrorCode::NO_AVAILABLE_HANDLE); +} + +} // namespace +} // namespace mooncake diff --git a/mooncake-store/tests/vchunk_master_service_test.cpp b/mooncake-store/tests/vchunk_master_service_test.cpp new file mode 100644 index 0000000000..091e13e21a --- /dev/null +++ b/mooncake-store/tests/vchunk_master_service_test.cpp @@ -0,0 +1,151 @@ +#include "master_service.h" + +#include + +#include +#include +#include +#include + +#include "master_config.h" +#include "types.h" + +namespace mooncake { +namespace { + +Segment MakeVChunkSegment(const std::string& name, uintptr_t base) { + Segment segment; + segment.id = generate_uuid(); + segment.name = name; + segment.base = base; + segment.size = 64U * 1024U * 1024U; + segment.te_endpoint = name; + return segment; +} + +TEST(VChunkMasterServiceTest, BuilderPropagatesVChunkConfiguration) { + VChunkConfig vchunk_config; + vchunk_config.enabled = true; + vchunk_config.max_slice_count = 128; + auto metadata_store = std::make_shared(); + const auto config = MasterServiceConfig::builder() + .set_vchunk_config(vchunk_config) + .set_vchunk_metadata_store(metadata_store) + .build(); + EXPECT_TRUE(config.vchunk_config.enabled); + EXPECT_EQ(config.vchunk_config.max_slice_count, 128U); + EXPECT_EQ(config.vchunk_metadata_store, metadata_store); +} + +TEST(VChunkMasterServiceTest, DisabledModeRejectsAllControlPlaneOperations) { + MasterServiceConfig config; + config.vchunk_config.enabled = false; + MasterService service(config); + const TenantId tenant("tenant"); + + EXPECT_EQ(service.VChunkPutStart(tenant, "key", 4096, false, 1).error(), + ErrorCode::UNAVAILABLE_IN_CURRENT_MODE); + EXPECT_EQ(service.VChunkPutEnd(tenant, "key", "id", 2), + ErrorCode::UNAVAILABLE_IN_CURRENT_MODE); + EXPECT_EQ(service.VChunkPutRevoke(tenant, "key", "id"), + ErrorCode::UNAVAILABLE_IN_CURRENT_MODE); + EXPECT_EQ(service.GetVChunk(tenant, "key").error(), + ErrorCode::UNAVAILABLE_IN_CURRENT_MODE); + EXPECT_EQ(service.AcquireVChunkRead(tenant, "key").error(), + ErrorCode::UNAVAILABLE_IN_CURRENT_MODE); + EXPECT_EQ(service.RemoveVChunk(tenant, "key", 3), + ErrorCode::UNAVAILABLE_IN_CURRENT_MODE); + EXPECT_EQ(service.ReapExpiredVChunks(4, 1).error(), + ErrorCode::UNAVAILABLE_IN_CURRENT_MODE); +} + +TEST(VChunkMasterServiceTest, PartitionedModeRejectsBeforeSlotViewIsReady) { + MasterServiceConfig config; + config.enable_ha = true; + config.ha_backend_type = "etcd"; + config.submaster_count = 2; + config.vchunk_config.enabled = true; + MasterService service(config); + + EXPECT_EQ(service + .VChunkPutStart(TenantId("tenant"), "key", 4096, false, 1) + .error(), + ErrorCode::SLOT_NOT_OWNED); + EXPECT_EQ(service.GetVChunk(TenantId("tenant"), "key").error(), + ErrorCode::SLOT_NOT_OWNED); +} + +TEST(VChunkMasterServiceTest, BackgroundReaperStopsAndCleansExpiredCreating) { + MasterServiceConfig config; + config.memory_allocator = BufferAllocatorType::OFFSET; + config.vchunk_config.enabled = true; + config.vchunk_config.creating_timeout_ms = 1; + config.vchunk_config.reaper_interval_ms = 5; + config.vchunk_config.reaper_max_scan = 4; + MasterService service(config); + ASSERT_TRUE(service + .MountSegment( + MakeVChunkSegment("reaper-segment", 0xE00000000ULL), + generate_uuid()) + .has_value()); + const TenantId tenant("tenant"); + ASSERT_TRUE( + service.VChunkPutStart(tenant, "expired", 4096, false, 0).has_value()); + for (int i = 0; i < 100; ++i) { + if (service.GetVChunk(tenant, "expired").error() == + ErrorCode::OBJECT_NOT_FOUND) { + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + EXPECT_EQ(service.GetVChunk(tenant, "expired").error(), + ErrorCode::OBJECT_NOT_FOUND); + const auto metrics = service.GetVChunkMetrics(); + EXPECT_EQ(metrics.states[static_cast(VChunkStatus::CREATING)], 0U); + EXPECT_GE(metrics.rollbacks, 1U); +} + +TEST(VChunkMasterServiceTest, ExposesIsolatedVChunkControlPlane) { + MasterServiceConfig config; + config.memory_allocator = BufferAllocatorType::OFFSET; + config.vchunk_config.enabled = true; + MasterService service(config); + const auto client_id = generate_uuid(); + ASSERT_TRUE(service + .MountSegment( + MakeVChunkSegment("vchunk-segment-a", 0x900000000ULL), + client_id) + .has_value()); + ASSERT_TRUE(service + .MountSegment( + MakeVChunkSegment("vchunk-segment-b", 0xA00000000ULL), + client_id) + .has_value()); + + const TenantId tenant("tenant-a"); + auto created = + service.VChunkPutStart(tenant, "key", 10U * 1024U, false, 100); + ASSERT_TRUE(created.has_value()); + EXPECT_EQ(created->row_size, 2U); + EXPECT_EQ(service.VChunkPutEnd(tenant, "key", created->vchunk_id, 200), + ErrorCode::OK); + + auto active = service.GetVChunk(tenant, "key"); + ASSERT_TRUE(active.has_value()); + EXPECT_EQ(active->status, VChunkStatus::ACTIVE); + auto remote_read = service.AcquireVChunkReadLease(tenant, "key", 250); + ASSERT_TRUE(remote_read.has_value()); + EXPECT_FALSE(remote_read->lease_id.empty()); + EXPECT_EQ(remote_read->record.vchunk_id, created->vchunk_id); + EXPECT_EQ(service.RemoveVChunk(tenant, "key", 300), ErrorCode::OK); + EXPECT_EQ(service.ReleaseVChunkReadLease(remote_read->lease_id), + ErrorCode::OK); + EXPECT_EQ(service.ReleaseVChunkReadLease(remote_read->lease_id), + ErrorCode::OK); + EXPECT_EQ(service.ReleaseVChunkReadLease(""), ErrorCode::INVALID_PARAMS); + EXPECT_EQ(service.GetVChunk(tenant, "key").error(), + ErrorCode::OBJECT_NOT_FOUND); +} + +} // namespace +} // namespace mooncake diff --git a/mooncake-store/tests/vchunk_metadata_store_test.cpp b/mooncake-store/tests/vchunk_metadata_store_test.cpp new file mode 100644 index 0000000000..175386a36b --- /dev/null +++ b/mooncake-store/tests/vchunk_metadata_store_test.cpp @@ -0,0 +1,278 @@ +#include "vchunk_metadata_store.h" + +#include + +#include +#include +#include +#include + +#include "allocation_strategy.h" +#include "master_service.h" +#include "vchunk_master_manager.h" +#include "vchunk_test_allocator.h" + +namespace mooncake { +namespace { + +class FaultStore final : public VChunkMetadataStore { + public: + ErrorCode Put(const VChunkMetadataRecord& record) override { + if (fail_put) return ErrorCode::ETCD_OPERATION_ERROR; + for (auto& current : records) { + if (current.vchunk_id == record.vchunk_id) { + current = record; + return ErrorCode::OK; + } + } + records.push_back(record); + return ErrorCode::OK; + } + ErrorCode Remove(const VChunkMetadataRecord& record) override { + if (fail_remove) return ErrorCode::ETCD_OPERATION_ERROR; + std::erase_if(records, [&](const auto& current) { + return current.vchunk_id == record.vchunk_id; + }); + return ErrorCode::OK; + } + tl::expected, ErrorCode> List() override { + if (fail_list) { + return tl::make_unexpected(ErrorCode::ETCD_OPERATION_ERROR); + } + return records; + } + bool IsPersistent() const override { return true; } + + bool fail_put{false}; + bool fail_remove{false}; + bool fail_list{false}; + std::vector records; +}; + +struct StoreFixture { + AllocatorManager allocators; + std::shared_ptr allocator = + std::make_shared( + "segment", 0xD00000000ULL, 4U * 1024U * 1024U); + std::shared_ptr store = std::make_shared(); + VChunkConfig config; + + StoreFixture() { + allocators.addAllocator("segment", allocator); + config.enabled = true; + config.creating_timeout_ms = 100; + } +}; + +TEST(VChunkMetadataStoreTest, UsesStableTenantAndVChunkNamespace) { + VChunkMetadataRecord record; + record.tenant_id = "tenant-a"; + record.vchunk_id = "id-a"; + EXPECT_EQ(MakeVChunkMetadataStoreKey(record), + "/mooncake/vchunk/v1/tenant-a/id-a"); +} + +TEST(VChunkMetadataStoreTest, PutStartIsNotVisibleWhenDurableWriteFails) { + StoreFixture fixture; + fixture.store->fail_put = true; + VChunkMasterManager manager(fixture.config, fixture.store); + auto result = manager.PutStart(fixture.allocators, TenantId("tenant"), + "key", 4096, false, 10); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error(), ErrorCode::ETCD_OPERATION_ERROR); + EXPECT_EQ(manager.SizeForTesting(), 0U); + EXPECT_EQ(fixture.allocator->size(), 0U); + + fixture.store->fail_put = false; + EXPECT_TRUE(manager.PutStart(fixture.allocators, TenantId("tenant"), + "key", 4096, false, 11) + .has_value()); +} + +TEST(VChunkMetadataStoreTest, PersistsCreatingThenActiveBeforeVisibility) { + StoreFixture fixture; + VChunkMasterManager manager(fixture.config, fixture.store); + auto created = manager.PutStart(fixture.allocators, TenantId("tenant"), + "key", 4096, false, 10); + ASSERT_TRUE(created.has_value()); + ASSERT_EQ(fixture.store->records.size(), 1U); + EXPECT_EQ(fixture.store->records[0].status, VChunkStatus::CREATING); + ASSERT_EQ(manager.PutEnd(TenantId("tenant"), "key", created->vchunk_id, + 20), + ErrorCode::OK); + EXPECT_EQ(fixture.store->records[0].status, VChunkStatus::ACTIVE); +} + +TEST(VChunkMetadataStoreTest, RecoveryRejectsActiveWithoutAllocatorRestore) { + StoreFixture fixture; + VChunkMetadataRecord active; + { + VChunkMasterManager writer(fixture.config, fixture.store); + auto created = writer.PutStart(fixture.allocators, TenantId("tenant"), + "active", 4096, false, 10); + ASSERT_TRUE(created.has_value()); + ASSERT_EQ(writer.PutEnd(TenantId("tenant"), "active", + created->vchunk_id, 20), + ErrorCode::OK); + active = *writer.Get(TenantId("tenant"), "active"); + } + auto expired = active; + expired.vchunk_id = "expired"; + expired.key = "expired"; + expired.status = VChunkStatus::CREATING; + expired.last_updated_at_ms = 10; + for (auto& slice : expired.slices) { + slice.status = VCSliceStatus::PENDING; + } + ASSERT_EQ(fixture.store->Put(expired), ErrorCode::OK); + + VChunkMasterManager recovered(fixture.config, fixture.store); + EXPECT_EQ(recovered.Recover(200), ErrorCode::REPLICA_IS_GONE); + EXPECT_EQ(recovered.SizeForTesting(), 0U); + // Validation is atomic: no incomplete record is removed when an ACTIVE + // record makes the whole snapshot unsafe to restore. + ASSERT_EQ(fixture.store->List()->size(), 2U); +} + +TEST(VChunkMetadataStoreTest, RecoveryCleansIncompleteWrites) { + StoreFixture fixture; + VChunkMasterManager writer(fixture.config, fixture.store); + ASSERT_TRUE(writer.PutStart(fixture.allocators, TenantId("tenant"), + "creating", 4096, false, 10) + .has_value()); + + VChunkMasterManager recovered(fixture.config, fixture.store); + EXPECT_EQ(recovered.Recover(20), ErrorCode::OK); + EXPECT_TRUE(fixture.store->List()->empty()); +} + +TEST(VChunkMetadataStoreTest, RecoveryOnlyProcessesOwnedSubmasterRecords) { + StoreFixture fixture; + VChunkMetadataRecord foreign_active; + { + VChunkMasterManager writer(fixture.config, fixture.store); + auto foreign = writer.PutStart(fixture.allocators, TenantId("tenant"), + "foreign", 4096, false, 10); + ASSERT_TRUE(foreign.has_value()); + ASSERT_EQ(writer.PutEnd(TenantId("tenant"), "foreign", + foreign->vchunk_id, 20), + ErrorCode::OK); + foreign_active = *writer.Get(TenantId("tenant"), "foreign"); + } + + auto owned_incomplete = foreign_active; + owned_incomplete.vchunk_id = "owned-incomplete"; + owned_incomplete.key = "owned"; + owned_incomplete.status = VChunkStatus::CREATING; + owned_incomplete.last_updated_at_ms = 10; + for (auto& slice : owned_incomplete.slices) { + slice.status = VCSliceStatus::PENDING; + } + ASSERT_EQ(fixture.store->Put(owned_incomplete), ErrorCode::OK); + + VChunkMasterManager recovered(fixture.config, fixture.store); + EXPECT_EQ(recovered.Recover( + 100, [](const VChunkMetadataRecord& record) { + return record.key == "owned"; + }), + ErrorCode::OK); + auto records = fixture.store->List(); + ASSERT_TRUE(records.has_value()); + ASSERT_EQ(records->size(), 1U); + EXPECT_EQ(records->front().vchunk_id, foreign_active.vchunk_id); +} + +TEST(VChunkMetadataStoreTest, ReaperIsBoundedAndRetryable) { + StoreFixture fixture; + VChunkMasterManager manager(fixture.config, fixture.store); + for (int i = 0; i < 3; ++i) { + ASSERT_TRUE(manager.PutStart(fixture.allocators, TenantId("tenant"), + "key-" + std::to_string(i), 4096, false, + 10) + .has_value()); + } + auto first = manager.ReapExpired(200, 2); + ASSERT_TRUE(first.has_value()); + EXPECT_EQ(*first, 2U); + EXPECT_EQ(manager.SizeForTesting(), 1U); + fixture.store->fail_remove = true; + EXPECT_EQ(manager.ReapExpired(200, 2).error(), + ErrorCode::ETCD_OPERATION_ERROR); + EXPECT_EQ(manager.SizeForTesting(), 1U); + fixture.store->fail_remove = false; + EXPECT_EQ(*manager.ReapExpired(200, 2), 1U); + EXPECT_EQ(manager.SizeForTesting(), 0U); +} + +TEST(VChunkMetadataStoreTest, ReaperSkipsRecordsOwnedByAnotherSubmaster) { + StoreFixture fixture; + VChunkMasterManager manager(fixture.config, fixture.store); + ASSERT_TRUE(manager.PutStart(fixture.allocators, TenantId("tenant"), + "owned", 4096, false, 10) + .has_value()); + ASSERT_TRUE(manager.PutStart(fixture.allocators, TenantId("tenant"), + "foreign", 4096, false, 10) + .has_value()); + + auto result = manager.ReapExpired( + 200, 2, [](const VChunkMetadataRecord& record) { + return record.key == "owned"; + }); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(*result, 1U); + EXPECT_EQ(manager.SizeForTesting(), 1U); + EXPECT_EQ(manager.Get(TenantId("tenant"), "foreign")->key, "foreign"); +} + +TEST(VChunkMetadataStoreTest, RemoveFailureLeavesReleasingObjectRetryable) { + StoreFixture fixture; + VChunkMasterManager manager(fixture.config, fixture.store); + const TenantId tenant("tenant"); + auto created = manager.PutStart(fixture.allocators, tenant, "key", 4096, + false, 10); + ASSERT_TRUE(created.has_value()); + ASSERT_EQ(manager.PutEnd(tenant, "key", created->vchunk_id, 20), + ErrorCode::OK); + fixture.store->fail_remove = true; + EXPECT_EQ(manager.Remove(tenant, "key", 30), + ErrorCode::ETCD_OPERATION_ERROR); + EXPECT_EQ(manager.Get(tenant, "key").error(), + ErrorCode::REPLICA_IS_NOT_READY); + fixture.store->fail_remove = false; + EXPECT_EQ(manager.Remove(tenant, "key", 31), ErrorCode::OK); + EXPECT_EQ(manager.SizeForTesting(), 0U); +} + +TEST(VChunkMetadataStoreTest, StartupRejectsUnavailableStore) { + auto store = std::make_shared(); + store->fail_list = true; + MasterServiceConfig unavailable; + unavailable.vchunk_config.enabled = true; + unavailable.vchunk_metadata_store = store; + EXPECT_THROW(MasterService service(unavailable), std::runtime_error); +} + +TEST(VChunkMetadataStoreTest, StartupAllowsVChunkWithHaEnabled) { + MasterServiceConfig coexistence; + coexistence.vchunk_config.enabled = true; + coexistence.enable_ha = true; + + MasterService service(coexistence); + EXPECT_TRUE(service.GetVChunkRuntimeInfo().enabled); +} + +TEST(VChunkMetadataStoreTest, RecoveryRejectsUnknownSchemaVersion) { + StoreFixture fixture; + VChunkMetadataRecord record; + record.schema_version = kVChunkMetadataSchemaVersion + 1; + record.vchunk_id = "future"; + record.tenant_id = "tenant"; + record.key = "key"; + fixture.store->records.push_back(record); + VChunkMasterManager manager(fixture.config, fixture.store); + EXPECT_EQ(manager.Recover(100), ErrorCode::INVALID_VERSION); + EXPECT_EQ(manager.SizeForTesting(), 0U); +} + +} // namespace +} // namespace mooncake diff --git a/mooncake-store/tests/vchunk_metadata_test.cpp b/mooncake-store/tests/vchunk_metadata_test.cpp new file mode 100644 index 0000000000..e98c1e5c45 --- /dev/null +++ b/mooncake-store/tests/vchunk_metadata_test.cpp @@ -0,0 +1,141 @@ +#include "vchunk_metadata.h" + +#include + +#include +#include +#include +#include + +namespace mooncake { +namespace { + +VChunkMetadataRecord MakeValidRecord() { + VChunkMetadataRecord record; + record.vchunk_id = "vchunk-1"; + record.tenant_id = "tenant-1"; + record.key = "key-1"; + record.total_size = 10U * 1024U; + record.slice_count = 3; + record.slice_size_level = VCSliceSizeLevel::k4K; + record.row_size = 2; + record.status = VChunkStatus::CREATING; + record.created_at_ms = 100; + record.last_updated_at_ms = 100; + record.slices = { + VCSliceDescriptor{0, "segment-a", 0, 4096, 4096, + VCSliceStatus::PENDING, 0}, + VCSliceDescriptor{1, "segment-b", 0, 4096, 4096, + VCSliceStatus::PENDING, 0}, + VCSliceDescriptor{2, "segment-a", 4096, 2048, 4096, + VCSliceStatus::PENDING, 0}, + }; + return record; +} + +TEST(VChunkMetadataTest, AcceptsValidLayout) { + EXPECT_EQ(ValidateVChunkMetadata(MakeValidRecord(), VChunkConfig{}), + ErrorCode::OK); +} + +TEST(VChunkMetadataTest, RoundTripsStableRecord) { + const auto original = MakeValidRecord(); + auto serialized = SerializeVChunkMetadata(original, VChunkConfig{}); + ASSERT_TRUE(serialized.has_value()); + + auto restored = + DeserializeVChunkMetadata(serialized.value(), VChunkConfig{}); + ASSERT_TRUE(restored.has_value()); + EXPECT_EQ(restored->schema_version, original.schema_version); + EXPECT_EQ(restored->vchunk_id, original.vchunk_id); + EXPECT_EQ(restored->tenant_id, original.tenant_id); + EXPECT_EQ(restored->key, original.key); + EXPECT_EQ(restored->total_size, original.total_size); + EXPECT_EQ(restored->slices.size(), original.slices.size()); + EXPECT_EQ(restored->slices.back().logical_length, 2048U); + EXPECT_EQ(restored->created_at_ms, 100); +} + +TEST(VChunkMetadataTest, RejectsUnsupportedSchema) { + auto record = MakeValidRecord(); + record.schema_version = kVChunkMetadataSchemaVersion + 1; + EXPECT_EQ(ValidateVChunkMetadata(record, VChunkConfig{}), + ErrorCode::INVALID_VERSION); +} + +TEST(VChunkMetadataTest, RejectsGapsAndDuplicateSegmentsWithinRow) { + auto record = MakeValidRecord(); + record.slices[1].slice_index = 3; + EXPECT_EQ(ValidateVChunkMetadata(record, VChunkConfig{}), + ErrorCode::INVALID_PARAMS); + + record = MakeValidRecord(); + record.slices[1].target_segment_name = "segment-a"; + EXPECT_EQ(ValidateVChunkMetadata(record, VChunkConfig{}), + ErrorCode::INVALID_PARAMS); +} + +TEST(VChunkMetadataTest, RejectsIncorrectCoverageAndAllocation) { + auto record = MakeValidRecord(); + record.slices.back().logical_length = 1024; + EXPECT_EQ(ValidateVChunkMetadata(record, VChunkConfig{}), + ErrorCode::INVALID_PARAMS); + + record = MakeValidRecord(); + record.slices.back().allocated_length = 1024; + EXPECT_EQ(ValidateVChunkMetadata(record, VChunkConfig{}), + ErrorCode::INVALID_PARAMS); +} + +TEST(VChunkMetadataTest, RejectsOffsetOverflowAndRetryOverflow) { + auto record = MakeValidRecord(); + record.slices[0].target_offset = + std::numeric_limits::max() - 1024; + EXPECT_EQ(ValidateVChunkMetadata(record, VChunkConfig{}), + ErrorCode::INVALID_PARAMS); + + record = MakeValidRecord(); + record.slices[0].retry_count = VChunkConfig{}.max_slice_retry + 1; + EXPECT_EQ(ValidateVChunkMetadata(record, VChunkConfig{}), + ErrorCode::INVALID_PARAMS); +} + +TEST(VChunkMetadataTest, EnforcesMetadataSizeLimit) { + auto config = VChunkConfig{}; + config.max_metadata_bytes = 8; + auto serialized = SerializeVChunkMetadata(MakeValidRecord(), config); + ASSERT_FALSE(serialized.has_value()); + EXPECT_EQ(serialized.error(), ErrorCode::BUFFER_OVERFLOW); +} + +TEST(VChunkMetadataTest, RejectsCorruptedBytes) { + auto serialized = + SerializeVChunkMetadata(MakeValidRecord(), VChunkConfig{}); + ASSERT_TRUE(serialized.has_value()); + serialized->pop_back(); + + auto restored = + DeserializeVChunkMetadata(serialized.value(), VChunkConfig{}); + EXPECT_FALSE(restored.has_value()); + EXPECT_EQ(restored.error(), ErrorCode::INVALID_PARAMS); +} + +TEST(VChunkMetadataTest, RuntimeStateTransitionsAreValidated) { + VChunkMetadata metadata(MakeValidRecord()); + EXPECT_EQ(metadata.TransitionTo(VChunkStatus::ACTIVE, 200), ErrorCode::OK); + EXPECT_EQ(metadata.TransitionTo(VChunkStatus::CREATING, 300), + ErrorCode::INVALID_PARAMS); + EXPECT_EQ(metadata.TransitionTo(VChunkStatus::RELEASING, 199), + ErrorCode::INVALID_PARAMS); + EXPECT_EQ(metadata.TransitionTo(VChunkStatus::RELEASING, 300), + ErrorCode::OK); + EXPECT_EQ(metadata.TransitionTo(VChunkStatus::RELEASED, 400), + ErrorCode::OK); + + const auto snapshot = metadata.Snapshot(); + EXPECT_EQ(snapshot.status, VChunkStatus::RELEASED); + EXPECT_EQ(snapshot.last_updated_at_ms, 400); +} + +} // namespace +} // namespace mooncake diff --git a/mooncake-store/tests/vchunk_test_allocator.h b/mooncake-store/tests/vchunk_test_allocator.h new file mode 100644 index 0000000000..d6834f3c0e --- /dev/null +++ b/mooncake-store/tests/vchunk_test_allocator.h @@ -0,0 +1,59 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "allocator.h" + +namespace mooncake::test { + +class VChunkTestAllocator + : public BufferAllocatorBase, + public std::enable_shared_from_this { + public: + VChunkTestAllocator(std::string segment_name, uintptr_t base, + size_t capacity) + : segment_name_(std::move(segment_name)), + base_(base), + capacity_(capacity) {} + + std::unique_ptr allocate(size_t size) override { + size_t current = used_.load(); + while (current <= capacity_ && size <= capacity_ - current) { + if (used_.compare_exchange_weak(current, current + size)) { + const auto offset = next_offset_.fetch_add(size); + return std::make_unique( + shared_from_this(), reinterpret_cast(base_ + offset), + size); + } + } + return nullptr; + } + + void deallocate(AllocatedBuffer* handle) override { + used_.fetch_sub(handle->size()); + } + + size_t capacity() const override { return capacity_; } + size_t size() const override { return used_.load(); } + std::string getSegmentName() const override { return segment_name_; } + std::string getTransportEndpoint() const override { + return segment_name_; + } + size_t getLargestFreeRegion() const override { + const auto used = used_.load(); + return used < capacity_ ? capacity_ - used : 0; + } + + private: + std::string segment_name_; + uintptr_t base_; + size_t capacity_; + std::atomic used_{0}; + std::atomic next_offset_{0}; +}; + +} // namespace mooncake::test diff --git a/mooncake-store/tests/vchunk_transfer_engine_test.cpp b/mooncake-store/tests/vchunk_transfer_engine_test.cpp new file mode 100644 index 0000000000..ccaa57f388 --- /dev/null +++ b/mooncake-store/tests/vchunk_transfer_engine_test.cpp @@ -0,0 +1,98 @@ +#include "vchunk_transfer_engine.h" + +#include + +#include +#include + +namespace mooncake { +namespace { + +VChunkMetadataRecord MakeRecord() { + VChunkMetadataRecord record; + record.vchunk_id = "id"; + record.tenant_id = "tenant"; + record.key = "key"; + record.total_size = 4096 + 17; + record.slice_count = 2; + record.slice_size_level = VCSliceSizeLevel::k4K; + record.row_size = 2; + record.status = VChunkStatus::CREATING; + record.created_at_ms = 1; + record.last_updated_at_ms = 1; + record.slices = { + {0, "a", 1000, 4096, 4096, VCSliceStatus::PENDING, 0}, + {1, "b", 2000, 17, 4096, VCSliceStatus::PENDING, 0}}; + return record; +} + +TEST(VChunkTransferEngineTest, BuildsAllRequestsBeforeSubmission) { + auto record = MakeRecord(); + std::array buffer{}; + int resolutions = 0; + auto requests = BuildVChunkTransferRequests( + record, buffer.data(), buffer.size(), TransferRequest::WRITE, + [&](const std::string& name) + -> tl::expected { + ++resolutions; + return name == "a" ? 11 : 22; + }); + ASSERT_TRUE(requests.has_value()); + ASSERT_EQ(requests->size(), 2U); + EXPECT_EQ(resolutions, 2); + EXPECT_EQ((*requests)[0].source, buffer.data()); + EXPECT_EQ((*requests)[0].target_id, 11U); + EXPECT_EQ((*requests)[0].target_offset, 1000U); + EXPECT_EQ((*requests)[0].length, 4096U); + EXPECT_EQ((*requests)[1].source, buffer.data() + 4096); + EXPECT_EQ((*requests)[1].target_id, 22U); + EXPECT_EQ((*requests)[1].length, 17U); +} + +TEST(VChunkTransferEngineTest, RejectsAnyUnresolvableSliceAsAWhole) { + auto record = MakeRecord(); + std::array buffer{}; + auto requests = BuildVChunkTransferRequests( + record, buffer.data(), buffer.size(), TransferRequest::READ, + [](const std::string& name) + -> tl::expected { + if (name == "b") { + return tl::make_unexpected(ErrorCode::SEGMENT_NOT_FOUND); + } + return 11; + }); + ASSERT_FALSE(requests.has_value()); + EXPECT_EQ(requests.error(), ErrorCode::SEGMENT_NOT_FOUND); +} + +TEST(VChunkTransferEngineTest, ReusesHandleWithinOneSegment) { + auto record = MakeRecord(); + record.slices[1].target_segment_name = "a"; + record.row_size = 1; + std::array buffer{}; + int resolutions = 0; + auto requests = BuildVChunkTransferRequests( + record, buffer.data(), buffer.size(), TransferRequest::WRITE, + [&](const std::string&) + -> tl::expected { + ++resolutions; + return 11; + }); + ASSERT_TRUE(requests.has_value()); + EXPECT_EQ(resolutions, 1); +} + +TEST(VChunkTransferEngineTest, RejectsUnknownMetadataVersion) { + auto record = MakeRecord(); + record.schema_version = kVChunkMetadataSchemaVersion + 1; + std::array buffer{}; + auto requests = BuildVChunkTransferRequests( + record, buffer.data(), buffer.size(), TransferRequest::WRITE, + [](const std::string&) + -> tl::expected { return 11; }); + ASSERT_FALSE(requests.has_value()); + EXPECT_EQ(requests.error(), ErrorCode::INVALID_VERSION); +} + +} // namespace +} // namespace mooncake