feat/dma memcpy

bin/cli/README.md

@@ -41,7 +41,7 @@ cargo run -p cli --release -- execute <PROGRAM.elf> [--private-input <FILE>] [--
 |---|---|
 | `--private-input <FILE>` | Pass private input bytes to the guest (read via `get_private_input()`). |
 | `--flamegraph <FILE>` | Generate folded-stack flamegraph output. See [Guest Program Flamegraphs](#guest-program-flamegraphs). |
-| `--cycles` | Count instructions during execution and print the dynamic instruction count. Also reports `Keccak calls` / `Ecsm calls` (accelerator syscall invocations). Combined with `--flamegraph`, the accelerator lines are omitted (the flamegraph path exposes no per-log data). |
+| `--cycles` | Count instructions during execution and print the dynamic instruction count. Also reports `Keccak calls` / `Ecsm calls` / `Dma calls` (accelerator syscall invocations), plus `Dma bytes` copied and the `Dma rows` those copies add to the trace before its power-of-two padding. One guest `memcpy` is chunked into several DMA ecalls, so the byte and row lines, not the call count, are what the copies cost. Combined with `--flamegraph`, the accelerator lines are omitted (the flamegraph path exposes no per-log data). |
 
 ### Prove
 

bin/cli/src/main.rs

@@ -11,7 +11,7 @@ use clap::{Parser, Subcommand, ValueHint};
 #[global_allocator]
 static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
 use executor::vm::instruction::decoding::Instruction;
-use executor::vm::instruction::execution::{Accelerator, SyscallNumbers};
+use executor::vm::instruction::execution::{Accelerator, SyscallNumbers, dma_memcpy_trace_rows};
 use executor::{elf::Elf, flamegraph::FlamegraphGenerator, vm::execution::Executor};
 use prover::VmProof;
 use stark::proof::options::GoldilocksCubicProofOptions;
@@ -142,9 +142,12 @@ enum Commands {
         cycle_budget: Option<u64>,
 
         /// Print the dynamic instruction (cycle) count, plus `Keccak calls` /
-        /// `Ecsm calls` (accelerator syscall invocations). The accelerator lines
-        /// are omitted when combined with --flamegraph (that path has no per-log
-        /// data).
+        /// `Ecsm calls` / `Dma calls` (accelerator syscall invocations), and for
+        /// DMA the `Dma bytes` copied and the `Dma rows` those copies add to the
+        /// trace before its power-of-two padding. One `memcpy` is chunked into
+        /// several DMA ecalls, so the byte and row lines, not the call count, are
+        /// what the copies cost. The accelerator lines are omitted when combined
+        /// with --flamegraph (that path has no per-log data).
         #[arg(long)]
         cycles: bool,
     },
@@ -359,6 +362,38 @@ struct FlamegraphCliOptions {
     checkpoint_cycles: Option<u64>,
 }
 
+/// One tally per [`Accelerator`] variant, printed by `execute --cycles`.
+#[derive(Default)]
+struct AccelCounts {
+    keccak: u64,
+    ecsm: u64,
+    dma: u64,
+    /// Bytes copied and DMA table rows those copies consume. Keccak and ECSM
+    /// cost the same per call, so DMA is the only accelerator whose report needs
+    /// a size next to its count: one `memcpy` becomes as many ecalls as the
+    /// guest stub chunks it into, which makes `dma` alone a poor cost proxy.
+    dma_bytes: u64,
+    dma_rows: u64,
+}
+
+impl AccelCounts {
+    /// Exhaustive `match`: a new `Accelerator` variant is a compile error here,
+    /// so it cannot be executed without also being reported. `dst_val` is the
+    /// ECALL's logged destination operand, which for DMA is the chunk's byte
+    /// count and for the other accelerators is unused.
+    fn tally(&mut self, accelerator: Accelerator, dst_val: u64) {
+        match accelerator {
+            Accelerator::Keccak => self.keccak += 1,
+            Accelerator::Ecsm => self.ecsm += 1,
+            Accelerator::Dma => {
+                self.dma += 1;
+                self.dma_bytes += dst_val;
+                self.dma_rows += dma_memcpy_trace_rows(dst_val);
+            }
+        }
+    }
+}
+
 /// Classifies one executed instruction as an accelerator syscall invocation.
 ///
 /// Delegates to the executor's canonical `SyscallNumbers::accelerator()` so the
@@ -412,7 +447,7 @@ fn cmd_execute(
     // below (the flamegraph path drives execution inside the executor and does
     // not expose per-log data). `None` means "not counted", so the accel lines
     // are omitted rather than printed as misleading zeros.
-    let mut accel_counts: Option<(u64, u64)> = None;
+    let mut accel_counts: Option<AccelCounts> = None;
 
     let cycle_count = if let Some(ref output_path) = flamegraph.path {
         // Shared execute+flamegraph path (executor::flamegraph) instead of
@@ -478,14 +513,13 @@ fn cmd_execute(
         };
 
         let mut cycle_count: u64 = 0;
-        let mut keccak_calls: u64 = 0;
-        let mut ecsm_calls: u64 = 0;
-        // Reused per chunk: `(current_pc, a7)` for logs whose a7 matches an
-        // accelerator syscall number. This is a cheap superset — a non-ECALL
+        let mut counts = AccelCounts::default();
+        // Reused per chunk: `(current_pc, a7, dst_val)` for logs whose a7 matches
+        // an accelerator syscall number. This is a cheap superset — a non-ECALL
         // instruction can hold the same value in src1 — that `accelerator_of`
         // confirms below, once the chunk's `&Log` borrow (tied to the executor's
         // `&mut`) is released so the instruction cache can be read again.
-        let mut accel_candidates: Vec<(u64, u64)> = Vec::new();
+        let mut accel_candidates: Vec<(u64, u64, u64)> = Vec::new();
         loop {
             let logs = match executor.resume_budgeted(cycle_count, cycle_budget) {
                 Ok(logs) => logs,
@@ -502,17 +536,15 @@ fn cmd_execute(
                         .map(|s| s.accelerator().is_some())
                         .unwrap_or(false)
                     {
-                        accel_candidates.push((log.current_pc, log.src1_val));
+                        accel_candidates.push((log.current_pc, log.src1_val, log.dst_val));
                     }
                 }
             }
             // `logs` is no longer used, so the executor's `&mut` borrow is free
             // and the instruction cache can be read to confirm each candidate.
-            for (pc, a7) in accel_candidates.drain(..) {
-                match accelerator_of(executor.instructions.get(pc), a7) {
-                    Some(Accelerator::Keccak) => keccak_calls += 1,
-                    Some(Accelerator::Ecsm) => ecsm_calls += 1,
-                    None => {}
+            for (pc, a7, dst_val) in accel_candidates.drain(..) {
+                if let Some(accelerator) = accelerator_of(executor.instructions.get(pc), a7) {
+                    counts.tally(accelerator, dst_val);
                 }
             }
             if cycle_budget.is_some_and(|budget| cycle_count >= budget) {
@@ -526,16 +558,19 @@ fn cmd_execute(
         }
 
         if cycles {
-            accel_counts = Some((keccak_calls, ecsm_calls));
+            accel_counts = Some(counts);
         }
         cycle_count
     };
 
     if cycles {
         println!("Cycles: {}", cycle_count);
-        if let Some((keccak_calls, ecsm_calls)) = accel_counts {
-            println!("Keccak calls: {}", keccak_calls);
-            println!("Ecsm calls: {}", ecsm_calls);
+        if let Some(counts) = accel_counts {
+            println!("Keccak calls: {}", counts.keccak);
+            println!("Ecsm calls: {}", counts.ecsm);
+            println!("Dma calls: {}", counts.dma);
+            println!("Dma bytes: {}", counts.dma_bytes);
+            println!("Dma rows: {}", counts.dma_rows);
         }
     }
 
@@ -1102,43 +1137,112 @@ mod tests {
         assert_eq!(continuation_epoch_size(20).unwrap(), 1 << 20);
     }
 
+    /// The chip each syscall must drive, written out here rather than read back
+    /// from `SyscallNumbers::accelerator()`. Comparing the CLI against the
+    /// executor alone would pass if both agreed on the wrong answer — a chip
+    /// demoted to `None` has to fail somewhere, and this is that somewhere.
+    ///
+    /// Cross-checked row by row against `SyscallNumbers::ALL`, which the
+    /// executor's macro generates from the enum, so a new syscall fails the test
+    /// until it gets a row here.
+    const EXPECTED_ACCELERATORS: &[(SyscallNumbers, Option<Accelerator>)] = &[
+        (SyscallNumbers::KeccakPermute, Some(Accelerator::Keccak)),
+        (SyscallNumbers::Ecsm, Some(Accelerator::Ecsm)),
+        (SyscallNumbers::DmaMemcpy, Some(Accelerator::Dma)),
+        (SyscallNumbers::Print, None),
+        (SyscallNumbers::Panic, None),
+        (SyscallNumbers::Commit, None),
+        (SyscallNumbers::Halt, None),
+        // `hint` drives its own HINT table, but the executor maps it to no
+        // `Accelerator`: the ecall adds no correctness constraint, so there is no
+        // accelerated work to attribute. `execute --cycles` reports no hint line.
+        (SyscallNumbers::Hint, None),
+    ];
+
     // `accelerator_of` must match the prover's `CpuOperation::from_log`: count an
     // invocation only when the instruction is an ECALL AND a7 is the accelerator
-    // syscall number. Covers both accelerators, the non-accelerator syscalls, a
-    // non-ECALL whose src1 collides with an accelerator number, and a cache miss.
+    // syscall number.
     #[test]
     fn accelerator_of_mirrors_prover_classification() {
-        use executor::vm::instruction::execution::{ECSM_SYSCALL_NUMBER, KECCAK_SYSCALL_NUMBER};
-
         let ecall = Instruction::EcallEbreak;
 
-        assert_eq!(
-            accelerator_of(Some(&ecall), KECCAK_SYSCALL_NUMBER),
-            Some(Accelerator::Keccak)
-        );
-        assert_eq!(
-            accelerator_of(Some(&ecall), ECSM_SYSCALL_NUMBER),
-            Some(Accelerator::Ecsm)
-        );
+        for &syscall in SyscallNumbers::ALL {
+            let rows = EXPECTED_ACCELERATORS
+                .iter()
+                .filter(|(listed, _)| *listed == syscall)
+                .count();
+            assert_eq!(
+                rows, 1,
+                "{syscall:?} needs exactly one row in EXPECTED_ACCELERATORS"
+            );
+        }
 
-        // Non-accelerator syscalls (Commit=64, Halt=93) count as neither.
-        assert_eq!(
-            accelerator_of(Some(&ecall), SyscallNumbers::Commit as u64),
-            None
-        );
-        assert_eq!(
-            accelerator_of(Some(&ecall), SyscallNumbers::Halt as u64),
-            None
-        );
+        for &(syscall, expected) in EXPECTED_ACCELERATORS {
+            assert_eq!(
+                accelerator_of(Some(&ecall), syscall.raw()),
+                expected,
+                "ECALL with a7 of {syscall:?} must classify as {expected:?}"
+            );
+            // A non-ECALL instruction whose src1 happens to equal a syscall a7
+            // must not count — this is the `f.ecall &&` guard the prover applies.
+            assert_eq!(
+                accelerator_of(Some(&Instruction::Fence), syscall.raw()),
+                None,
+                "non-ECALL with a7 of {syscall:?} must not count"
+            );
+            // No decoded instruction at the pc (cache miss) counts as neither.
+            assert_eq!(accelerator_of(None, syscall.raw()), None);
+        }
+    }
 
-        // A non-ECALL instruction whose src1 happens to equal an accelerator a7
-        // must not count — this is the `f.ecall &&` guard the prover applies.
-        assert_eq!(
-            accelerator_of(Some(&Instruction::Fence), KECCAK_SYSCALL_NUMBER),
-            None
-        );
+    // Every tallied accelerator gets its own counter: no two variants may share
+    // a field, and each must land in the one the report prints.
+    #[test]
+    fn accel_counts_tallies_each_accelerator_separately() {
+        for &(_, expected_accelerator) in EXPECTED_ACCELERATORS {
+            let Some(accelerator) = expected_accelerator else {
+                continue;
+            };
+            let mut counts = AccelCounts::default();
+            counts.tally(accelerator, 0);
+            assert_eq!(
+                counts.keccak + counts.ecsm + counts.dma,
+                1,
+                "{accelerator:?} must increment exactly one counter"
+            );
+            let expected = match accelerator {
+                Accelerator::Keccak => counts.keccak,
+                Accelerator::Ecsm => counts.ecsm,
+                Accelerator::Dma => counts.dma,
+            };
+            assert_eq!(
+                expected, 1,
+                "{accelerator:?} must increment its own counter"
+            );
+        }
+    }
+
+    // The byte and row lines are what make the DMA report a cost figure rather
+    // than a call count, so they must accumulate across chunked ecalls and use
+    // the executor's row formula — the same one trace generation sizes with.
+    #[test]
+    fn accel_counts_sizes_dma_calls() {
+        let mut counts = AccelCounts::default();
+        for bytes in [256, 256, 8, 3, 0] {
+            counts.tally(Accelerator::Dma, bytes);
+        }
 
-        // No decoded instruction at the pc (cache miss) counts as neither.
-        assert_eq!(accelerator_of(None, KECCAK_SYSCALL_NUMBER), None);
+        assert_eq!(counts.dma, 5, "every DMA ecall counts as one call");
+        assert_eq!(counts.dma_bytes, 523);
+        // 33 + 33 + 2 + 4 + 1: eight-byte rows, one row per tail byte, and a
+        // terminal row each, with the zero-byte ecall contributing only its
+        // terminal row.
+        assert_eq!(counts.dma_rows, 73);
+
+        // The other accelerators must leave the DMA size lines alone.
+        let mut others = AccelCounts::default();
+        others.tally(Accelerator::Keccak, 200);
+        others.tally(Accelerator::Ecsm, 32);
+        assert_eq!((others.dma, others.dma_bytes, others.dma_rows), (0, 0, 0));
     }
 }

docs/general_flow.md

@@ -18,3 +18,17 @@ The Lambda VM proves correct execution of a RISC-V (RV64IM) program against an i
 4. **Proof system** ([`crypto/stark/`](../crypto/stark/)) — commits to each table's trace via Merkle trees, samples challenges via Fiat-Shamir, and runs FRI for the low-degree test. Produces a `MultiProof`; the verifier replays the transcript and checks all AIR and lookup constraints.
 
 For a deeper dive into each component see the [proof system overview](./cryptography/proof_system.md).
+
+## Accelerated memory operations
+
+`memcpy` is accelerated: `lambda-vm-syscalls` exports it under its standard, unmangled C name, so both explicit calls and the copies the compiler emits implicitly (struct moves, slice copies, `Vec` growth) reach the DMA ecall with no guest source changes. Behaviour is identical to the C function for every input, including `n == 0` and any alignment of `dest`, `src` or `n`. `memmove`, `memset` and `memcmp` are not accelerated and fall back to the toolchain's `compiler-builtins` definitions.
+
+**Observability.** `cli execute <elf> --cycles` reports `Dma calls`, `Dma bytes` and `Dma rows`. The call line confirms the accelerator is engaged at all; the byte and row lines are the cost, since the guest stub chunks one `memcpy` into as many ecalls as it needs and each ecall adds one DMA row per eight-byte chunk, one per tail byte, and a terminal row. `Dma rows` is the raw row count the copies contribute, before the DMA trace is padded to a power-of-two height — for a guest with few copies the padded table is larger than the reported figure.
+
+**Aligned vs misaligned.** The chunk width comes from the bytes remaining, not from the alignment of `dest` or `src`, so the DMA table's own row count is the same either way — but the cost is not. Each eight-byte chunk emits two width-8 memory operations, one reading the source and one writing the destination, each at its address as given, and the memory argument routes each one by that address: an 8-aligned window sharing one old timestamp reaches MEMW_A (29 columns, one ALU `LT` range check), and anything else falls to the general MEMW table (49 columns, eight `LT` rows). The two sides are independent, so a copy can take the fast path on one end and not the other; and because the width is chosen from the bytes remaining alone, a side that starts misaligned stays misaligned for every chunk. A misaligned copy therefore commits strictly more cells than an aligned copy of the same length, which is what makes the aligned/misaligned split the standard recommends informative here. It is not reported: the accelerator statistics are derived from `Log`, whose two operand slots are already taken (`src2_val = src`, `dst_val = n`, and `n` is what yields the byte and row figures), so reporting the split needs those statistics to move into the executor. Left as a follow-up, and stated here rather than claimed as done.
+
+**Symbol resolution.** `compiler-builtins` defines `memcpy` *weakly*, and a linker extracts a static-archive member only to satisfy an *undefined* symbol — a weak definition already satisfies the reference, so a strong definition that lives in a member nothing else pulls in is dropped silently, with no duplicate-symbol diagnostic. Lambda VM therefore defines `memcpy` in [`syscalls/src/entrypoint.rs`](../syscalls/src/entrypoint.rs), the same object that defines `_start`, which every guest links unconditionally. That object is always extracted, so the strong definition is in the link graph from the start and overrides the weak one. No `--whole-archive` and no guest link flag is required, and resolution does not depend on archive order.
+
+**Deviation from the standard's scope clause.** The standard says the accelerated symbols "are exported from the vendor static library defined by the Static Library and Linker Script standard". Lambda VM has no such library: the guest interface is a Rust rlib (`lambda-vm-syscalls`), and `memcpy` is exported from its always-linked entrypoint object. The linking clause above is satisfied by mechanism (1); the packaging the scope clause assumes is not, and adopting it is a repo-wide decision rather than one this accelerator can make.
+
+Placing `memcpy` beside `_start` is insurance rather than a repair: defined in `syscalls.rs` it also resolved correctly, and not by luck — `_start` calls `sys_halt` from that module and it is not `#[inline]`, so every guest carries an undefined reference that forces the object out of the archive, whatever the guest itself names. What the move removes is the two things that guarantee rested on: `_start` continuing to call into `syscalls.rs`, and rustc's codegen-unit merging keeping the two modules together. Co-locating with `_start` — the one symbol the linker is obliged to resolve — makes the guarantee local instead, and `test_dma_memcpy_compiler_emitted_copies` (a guest that never names `memcpy`, asserting the DMA ecall count stays above zero) is what detects a regression, since a guest that falls back to the weak definition still produces correct output.

executor/Cargo.toml

@@ -14,6 +14,7 @@ ecsm = { path = "../crypto/ecsm" }
 k256 = { version = "0.13", default-features = false, features = ["arithmetic", "expose-field"] }
 
 [dev-dependencies]
+proptest = "1.9"
 # Test-only: the guest-side syscall crate re-declares the `hint` selectors as `usize`
 # and they must stay equal to the `u64` copies here (see `hint_selectors_match_the_guest`).
 # Unlike `crypto/crypto`'s and `ethrex-crypto`'s copies of this dep, it is NOT

executor/programs/rust/dma_memcpy_cases/.cargo/config.toml

@@ -0,0 +1,9 @@
+[target.riscv64im-lambda-vm-elf]
+rustflags = [
+  "--cfg", "getrandom_backend=\"custom\"",
+  "-C", "passes=lower-atomic"
+]
+
+[env]
+CC_riscv64im_lambda_vm_elf = "clang"
+CFLAGS_riscv64im_lambda_vm_elf = "--target=riscv64 -march=rv64im -mabi=lp64 --sysroot=/opt/lambda-vm-sysroot"