diff --git a/.github/workflows/ci_pull_request.yml b/.github/workflows/ci_pull_request.yml index 02d4723..aae405f 100644 --- a/.github/workflows/ci_pull_request.yml +++ b/.github/workflows/ci_pull_request.yml @@ -9,6 +9,7 @@ name: CI (Pull Request) on: pull_request: + types: [opened, reopened, synchronize, labeled, unlabeled] push: branches: ['main'] @@ -59,5 +60,6 @@ jobs: with: name: default profile: default + include_optional_scenarios: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-optional-scenarios') }} enable_reporting: false secrets: inherit diff --git a/.github/workflows/ci_run.yml b/.github/workflows/ci_run.yml index 54b6696..25ef0ec 100644 --- a/.github/workflows/ci_run.yml +++ b/.github/workflows/ci_run.yml @@ -20,6 +20,11 @@ on: description: 'Dependency profile declared in ci/dependency-profiles.json' required: true type: string + include_optional_scenarios: + description: 'Run optional extended scenarios after the core suite' + required: false + type: boolean + default: false enable_reporting: description: 'When true, file a GitHub issue with the scenario report' required: false @@ -44,7 +49,7 @@ on: jobs: foc-start-test: runs-on: ["self-hosted", "linux", "x64", "4xlarge+disk"] - timeout-minutes: 100 + timeout-minutes: 150 permissions: contents: read issues: write @@ -423,7 +428,12 @@ jobs: SKIP_REPORT_ON_PASS: ${{ inputs.skip_report_on_pass }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} SCENARIO_RUN_TYPE: ${{ inputs.name }} - run: python3 scenarios/run.py + run: | + if [[ "${{ inputs.include_optional_scenarios }}" == "true" ]]; then + python3 scenarios/run.py --include-optional + else + python3 scenarios/run.py + fi # Ensure scenario report exists even if tests didn't run (for issue reporting) - name: "EXEC: {Ensure scenario report exists}" diff --git a/README_ADVANCED.md b/README_ADVANCED.md index b02c5f2..f5b1b8d 100644 --- a/README_ADVANCED.md +++ b/README_ADVANCED.md @@ -1314,9 +1314,12 @@ Scenario tests are Python scripts that validate devnet state after startup. They ### Running scenarios ```bash -# Run all scenarios +# Run the core scenario suite python3 scenarios/run.py +# Include extended optional scenarios +python3 scenarios/run.py --include-optional + # Run a single scenario directly python3 scenarios/test_basic_balances.py @@ -1328,7 +1331,11 @@ Reports are written to `~/.foc-devnet/state/latest/scenario_report.md`. ### CI integration -Scenarios run automatically in CI after the devnet starts. On nightly runs (or manual dispatch with `reporting` enabled), failures automatically create a GitHub issue with a full report. +Scenarios run automatically in CI after the devnet starts. Pull requests run the +core suite by default. Apply the `run-optional-scenarios` label to rerun the +same pull-request CI with the extended scenarios included. On nightly runs (or +manual dispatch with `reporting` enabled), failures automatically create a +GitHub issue with a full report. CI resolves compatibility-sensitive dependencies from `ci/dependency-profiles.json`. Pull requests use the pinned `default` profile, while nightly `stability` runs use diff --git a/scenarios/report.py b/scenarios/report.py index facab24..9e726a9 100644 --- a/scenarios/report.py +++ b/scenarios/report.py @@ -59,6 +59,11 @@ def get_version_info(): ## Resolved dependencies $dependency_table +## Scenario selection +**$scenario_selection** + +$skipped_optional_tests + ## Tests summary $test_summary """) @@ -93,10 +98,25 @@ def _build_test_summary(results: list[TestResult]) -> str: return "\n\n".join(parts) -def write_report(results: list[TestResult] | None = None, elapsed: int = 0): +def _format_skipped_optional_tests(skipped_tests: list[str]) -> str: + if not skipped_tests: + return "No optional scenarios skipped." + return "Skipped optional scenarios:\n" + "\n".join( + f"- `{test_name}`" for test_name in skipped_tests + ) + + +def write_report( + results: list[TestResult] | None = None, + elapsed: int = 0, + selection: str = "core", + skipped_tests: list[str] | None = None, +): """Write a markdown report to REPORT_MD. Returns path written.""" if results is None: results = [] + if skipped_tests is None: + skipped_tests = [] total = len(results) passed = sum(1 for r in results if r.is_passed) content = _REPORT_TEMPLATE.substitute( @@ -108,6 +128,8 @@ def write_report(results: list[TestResult] | None = None, elapsed: int = 0): ci_run_link=_build_ci_run_link(), version_info=f"```\n{get_version_info()}\n```", dependency_table=format_markdown_table(), + scenario_selection=selection, + skipped_optional_tests=_format_skipped_optional_tests(skipped_tests), test_summary=_build_test_summary(results), ) with open(REPORT_MD, "w") as fh: diff --git a/scenarios/run.py b/scenarios/run.py index a89ce0c..fba0212 100755 --- a/scenarios/run.py +++ b/scenarios/run.py @@ -1,10 +1,12 @@ #!/usr/bin/env python3 """Scenario test runner — executes tests in order and generates a report. -Run all tests: python3 scenarios/run.py -Run one test: python3 scenarios/test_containers.py +Run core tests: python3 scenarios/run.py +Run core + optional: python3 scenarios/run.py --include-optional +Run one test: python3 scenarios/test_containers.py """ +import argparse import os import subprocess import sys @@ -22,20 +24,36 @@ from scenarios.report import TestResult, write_report # ── Scenario execution order ───────────────────────────────── -# Each entry is (test_name, timeout_seconds) +# Each entry is (test_name, timeout_seconds, optional). Optional scenarios are +# discoverable and runnable directly, but omitted from the default core run. CREATE_DATASET_SMOKE_TIMEOUT_SECS = 1800 ORDER = [ - ("test_containers", 5), - ("test_basic_balances", 10), + ("test_containers", 5, False), + ("test_basic_balances", 10, False), # Allows setup plus five 280s Node attempts and retry delays. - ("test_create_dataset_smoke", CREATE_DATASET_SMOKE_TIMEOUT_SECS), - ("test_synapse_e2e", 600), - ("test_multi_copy_upload", 600), - ("test_caching_subsystem", 200), + ("test_create_dataset_smoke", CREATE_DATASET_SMOKE_TIMEOUT_SECS, False), + ("test_synapse_e2e", 600, False), + ("test_negative_permissions", 300, False), + ("test_multi_copy_upload", 600, False), + ("test_caching_subsystem", 200, False), + ("test_bulk_add", 600, True), + ("test_termination_controls", 900, True), ] +def select_scenarios(include_optional, order=ORDER): + """Return (selected, skipped) entries for an ordered scenario collection.""" + selected = [] + skipped = [] + for scenario in order: + if scenario[2] and not include_optional: + skipped.append(scenario) + else: + selected.append(scenario) + return selected, skipped + + def _run_single_test(scenario_py_file, name, timeout_sec): """Run one scenario file as a subprocess, return a TestResult.""" info(f"=== {name} (timeout: {timeout_sec}s) ===") @@ -75,15 +93,26 @@ def _run_single_test(scenario_py_file, name, timeout_sec): ) -def run_tests(): - """Run scenarios in ORDER. Returns list of TestResult.""" +def run_tests(include_optional=False): + """Run selected scenarios in ORDER. Returns list of TestResult.""" pwd = os.path.dirname(os.path.abspath(__file__)) + selected, _ = select_scenarios(include_optional) return [ _run_single_test(os.path.join(pwd, f"{name}.py"), name, timeout) - for name, timeout in ORDER + for name, timeout, _ in selected ] +def _parse_args(): + parser = argparse.ArgumentParser(description="Run foc-devnet scenario tests") + parser.add_argument( + "--include-optional", + action="store_true", + help="run optional extended scenarios after the core scenarios", + ) + return parser.parse_args() + + def _print_summary(results, elapsed): """Print a human-readable summary to stdout.""" passed = sum(1 for r in results if r.is_passed) @@ -116,10 +145,21 @@ def _print_ci_url(): if __name__ == "__main__": + args = _parse_args() + _, skipped = select_scenarios(args.include_optional) + selection = "core + optional" if args.include_optional else "core" + info(f"Scenario selection: {selection}") + if skipped: + info( + "Skipping optional scenarios (use --include-optional): " + + ", ".join(name for name, _, _ in skipped) + ) start = time.time() - results = run_tests() + results = run_tests(args.include_optional) elapsed = int(time.time() - start) _print_summary(results, elapsed) - print(f"Report: {write_report(results=results)}") + print( + f"Report: {write_report(results=results, selection=selection, skipped_tests=[name for name, _, _ in skipped])}" + ) _print_ci_url() sys.exit(0 if all(r.is_passed for r in results) else 1) diff --git a/scenarios/synapse-e2e/bulk-add.ts b/scenarios/synapse-e2e/bulk-add.ts new file mode 100644 index 0000000..22d11a4 --- /dev/null +++ b/scenarios/synapse-e2e/bulk-add.ts @@ -0,0 +1,81 @@ +import assert from 'node:assert/strict' +import * as SP from '@filoz/synapse-core/sp' +import { findPieceIdsByCidCall, getActivePieceCount } from '@filoz/synapse-core/pdp-verifier' +import { getPdpDataSet } from '@filoz/synapse-core/warm-storage' +import { readContract } from 'viem/actions' +import { createSynapse, prepareAccount } from './account.ts' +import { freshMetadata, resolveEnvironment } from './environment.ts' +import { fileSize, uploadFile } from './storage.ts' + +const REQUIRED_PIECES = 40 +const SMALL_PIECE_BYTES = 64 * 1024 + +function fixtureFor(pieceNumber: number): File { + const bytes = Buffer.alloc(SMALL_PIECE_BYTES, pieceNumber % 251) + bytes.write(`foc-devnet-bulk-add-${pieceNumber}`, 0, 'utf8') + return new File([bytes], `bulk-${pieceNumber.toString().padStart(3, '0')}.bin`, { + type: 'application/octet-stream', + }) +} + +async function main(): Promise { + const environment = resolveEnvironment({ defaultUserIndex: 0, requireFiles: true }) + if (environment.filePaths.length !== 1) throw new Error('bulk-add.ts accepts exactly one bootstrap file path') + const [bootstrapPath] = environment.filePaths + const synapse = createSynapse(environment) + + // Prepare enough headroom for creation plus the maximum-size add-pieces batch. + await prepareAccount(synapse, (await fileSize(bootstrapPath)) + BigInt(SMALL_PIECE_BYTES * REQUIRED_PIECES)) + const { result } = await uploadFile(synapse, bootstrapPath, freshMetadata('bulk-add'), 1) + const copy = result.copies[0] + assert(copy != null, 'Expected a bootstrap data set') + const dataSet = await getPdpDataSet(synapse.client, { dataSetId: copy.dataSetId }) + assert(dataSet != null && dataSet.live, `Bootstrap data set ${copy.dataSetId} is not live`) + const initialPieceCount = await getActivePieceCount(synapse.client, { dataSetId: copy.dataSetId }) + + const files = Array.from({ length: REQUIRED_PIECES }, (_, index) => fixtureFor(index + 1)) + const added = await SP.upload(synapse.client, { + dataSetId: copy.dataSetId, + data: files, + }) + assert.equal(added.pieces.length, REQUIRED_PIECES, `Expected ${REQUIRED_PIECES} submitted pieces`) + const pieceCids = added.pieces.map((piece) => piece.pieceCid) + assert.equal( + new Set(pieceCids.map((pieceCid) => pieceCid.toString())).size, + REQUIRED_PIECES, + 'Piece CIDs are not unique' + ) + + const confirmed = await SP.waitForAddPieces({ statusUrl: added.statusUrl, timeout: 180_000, pollInterval: 1000 }) + assert.equal(confirmed.piecesAdded, true, 'Batched pieces were not added') + assert.equal(confirmed.confirmedPieceIds.length, REQUIRED_PIECES, 'Batched piece confirmation was incomplete') + console.log(`Added ${REQUIRED_PIECES} pieces in one batch: tx=${confirmed.txHash}`) + + const activePieceCount = await getActivePieceCount(synapse.client, { dataSetId: copy.dataSetId }) + assert.equal( + activePieceCount, + initialPieceCount + BigInt(REQUIRED_PIECES), + 'On-chain active-piece count differs from successfully submitted pieces' + ) + await Promise.all( + pieceCids.map(async (pieceCid) => { + const ids = await readContract( + synapse.client, + findPieceIdsByCidCall({ + chain: synapse.client.chain, + dataSetId: copy.dataSetId, + pieceCid, + startPieceId: 0n, + limit: 2n, + }) + ) + assert.equal(ids.length, 1, `Submitted piece ${pieceCid} is not discoverable on-chain`) + }) + ) + console.log(`=== SUCCESS: ${REQUIRED_PIECES} distinct pieces are discoverable on data set ${copy.dataSetId} ===`) +} + +main().catch((error: unknown) => { + console.error(error) + process.exitCode = 1 +}) diff --git a/scenarios/synapse-e2e/negative-permissions.ts b/scenarios/synapse-e2e/negative-permissions.ts new file mode 100644 index 0000000..075b667 --- /dev/null +++ b/scenarios/synapse-e2e/negative-permissions.ts @@ -0,0 +1,210 @@ +import assert from 'node:assert/strict' +import { + DeletePieceError, + TerminateServiceError, + WaitForTerminateServiceNotFoundError, + WaitForTerminateServiceRejectedError, +} from '@filoz/synapse-core/errors' +import * as SP from '@filoz/synapse-core/sp' +import { getRail } from '@filoz/synapse-core/pay' +import { getActivePieceCount } from '@filoz/synapse-core/pdp-verifier' +import { getPdpDataSet } from '@filoz/synapse-core/warm-storage' +import { waitForTransactionReceipt } from 'viem/actions' +import { createSynapse, prepareAccount, readAccountState } from './account.ts' +import { freshMetadata, resolveEnvironment } from './environment.ts' +import { fileSize, uploadFile } from './storage.ts' + +const NEGATIVE_TERMINATION_OBSERVATION_MS = 15_000 + +type DataSetSnapshot = { + dataSetId: bigint + pieceId: bigint + clientDataSetId: bigint + serviceURL: string + live: boolean + activePieceCount: bigint + rail: { railId: bigint; endEpoch: bigint; paymentRate: bigint; lockupPeriod: bigint } + payment: { funds: bigint; lockupRate: bigint } +} + +async function snapshot( + synapse: ReturnType, + dataSetId: bigint, + pieceId: bigint +): Promise { + const dataSet = await getPdpDataSet(synapse.client, { dataSetId }) + if (dataSet == null) throw new Error(`Data set ${dataSetId} is not readable`) + const [activePieceCount, rail, payment] = await Promise.all([ + getActivePieceCount(synapse.client, { dataSetId }), + getRail(synapse.client, { railId: dataSet.pdpRailId }), + readAccountState(synapse), + ]) + return { + dataSetId, + pieceId, + clientDataSetId: dataSet.clientDataSetId, + serviceURL: dataSet.provider.pdp.serviceURL, + live: dataSet.live, + activePieceCount, + rail: { + railId: dataSet.pdpRailId, + endEpoch: rail.endEpoch, + paymentRate: rail.paymentRate, + lockupPeriod: rail.lockupPeriod, + }, + payment: { + funds: payment.funds, + lockupRate: payment.lockupRate, + }, + } +} + +async function createPermissionTarget( + owner: ReturnType, + filePath: string, + label: string +): Promise { + const { result } = await uploadFile(owner, filePath, freshMetadata(`negative-permissions-${label}`), 1) + const copy = result.copies[0] + assert(copy != null, `Expected one live data set for ${label}`) + return snapshot(owner, copy.dataSetId, copy.pieceId) +} + +function assertStableAfterRejectedRequest(before: DataSetSnapshot, after: DataSetSnapshot, label: string): void { + assert.equal(after.live, true, `${label} mutated data set liveness`) + assert.equal(after.activePieceCount, before.activePieceCount, `${label} mutated active piece count`) + assert.deepEqual(after.rail, before.rail, `${label} mutated rail identity, rate, lockup period, or end epoch`) + assert.deepEqual(after.payment, before.payment, `${label} mutated payment funds or lockup rate`) +} + +async function assertTerminationRejected( + operation: () => Promise, + label: string +): Promise { + try { + const { statusUrl } = await operation() + await SP.waitForTerminateService({ + statusUrl, + timeout: NEGATIVE_TERMINATION_OBSERVATION_MS, + pollInterval: 1000, + }) + } catch (error) { + if ( + TerminateServiceError.is(error) || + WaitForTerminateServiceRejectedError.is(error) || + WaitForTerminateServiceNotFoundError.is(error) + ) { + console.log(`Rejected as expected: ${label}: ${error instanceof Error ? error.message : String(error)}`) + return + } + if (error instanceof Error && error.name === 'TimeoutError') { + console.log(`No successful termination observed for ${label} after bounded wait`) + return + } + throw error + } + throw new Error(`${label} unexpectedly terminated the data set`) +} + +async function assertDeletionRejected( + synapse: ReturnType, + operation: () => Promise, + label: string +): Promise { + try { + const { hash } = await operation() + const receipt = await waitForTransactionReceipt(synapse.client, { hash }) + assert.equal(receipt.status, 'reverted', `${label} transaction unexpectedly succeeded`) + console.log(`Rejected as expected: ${label}: reverted tx ${hash}`) + } catch (error) { + if (DeletePieceError.is(error)) { + console.log(`Rejected as expected: ${label}: ${error instanceof Error ? error.message : String(error)}`) + return + } + throw error + } +} + +async function main(): Promise { + const ownerEnvironment = resolveEnvironment({ defaultUserIndex: 0, requireFiles: true }) + if (ownerEnvironment.filePaths.length !== 1) throw new Error('negative-permissions.ts accepts exactly one file path') + const owner = createSynapse(ownerEnvironment) + const intruder = createSynapse(resolveEnvironment({ defaultUserIndex: 1 })) + const [filePath] = ownerEnvironment.filePaths + + await prepareAccount(owner, (await fileSize(filePath)) * 4n) + + const nonOwnerTerminate = await createPermissionTarget(owner, filePath, 'non-owner-terminate') + await assertTerminationRejected( + () => + SP.terminateService(intruder.client, { + serviceURL: nonOwnerTerminate.serviceURL, + dataSetId: nonOwnerTerminate.dataSetId, + }), + 'non-owner relayed termination' + ) + assertStableAfterRejectedRequest( + nonOwnerTerminate, + await snapshot(owner, nonOwnerTerminate.dataSetId, nonOwnerTerminate.pieceId), + 'non-owner relayed termination' + ) + + const malformedTerminate = await createPermissionTarget(owner, filePath, 'malformed-terminate') + await assertTerminationRejected( + () => + SP.terminateServiceApiRequest({ + serviceURL: malformedTerminate.serviceURL, + dataSetId: malformedTerminate.dataSetId, + extraData: '0x', + }), + 'malformed relayed termination data' + ) + assertStableAfterRejectedRequest( + malformedTerminate, + await snapshot(owner, malformedTerminate.dataSetId, malformedTerminate.pieceId), + 'malformed relayed termination data' + ) + + const nonOwnerDeletion = await createPermissionTarget(owner, filePath, 'non-owner-deletion') + await assertDeletionRejected( + owner, + () => + SP.schedulePieceDeletion(intruder.client, { + serviceURL: nonOwnerDeletion.serviceURL, + dataSetId: nonOwnerDeletion.dataSetId, + clientDataSetId: nonOwnerDeletion.clientDataSetId, + pieceId: nonOwnerDeletion.pieceId, + }), + 'non-owner piece deletion' + ) + assertStableAfterRejectedRequest( + nonOwnerDeletion, + await snapshot(owner, nonOwnerDeletion.dataSetId, nonOwnerDeletion.pieceId), + 'non-owner piece deletion' + ) + + const malformedDeletion = await createPermissionTarget(owner, filePath, 'malformed-deletion') + await assertDeletionRejected( + owner, + () => + SP.deletePiece({ + serviceURL: malformedDeletion.serviceURL, + dataSetId: malformedDeletion.dataSetId, + pieceId: malformedDeletion.pieceId, + extraData: '0x', + }), + 'malformed piece deletion data' + ) + assertStableAfterRejectedRequest( + malformedDeletion, + await snapshot(owner, malformedDeletion.dataSetId, malformedDeletion.pieceId), + 'malformed piece deletion data' + ) + + console.log('=== SUCCESS: rejected permission requests left independent data sets unchanged ===') +} + +main().catch((error: unknown) => { + console.error(error) + process.exitCode = 1 +}) diff --git a/scenarios/synapse-e2e/termination-controls.ts b/scenarios/synapse-e2e/termination-controls.ts new file mode 100644 index 0000000..3fd5c50 --- /dev/null +++ b/scenarios/synapse-e2e/termination-controls.ts @@ -0,0 +1,72 @@ +import assert from 'node:assert/strict' +import { getRail } from '@filoz/synapse-core/pay' +import { getPriceList, getPdpDataSet } from '@filoz/synapse-core/warm-storage' +import { createSynapse, prepareAccount, readAccountState } from './account.ts' +import { freshMetadata, resolveEnvironment } from './environment.ts' +import { fileSize, uploadFile } from './storage.ts' + +const delay = (milliseconds: number) => new Promise((resolve) => setTimeout(resolve, milliseconds)) + +async function waitForTerminatedRail(synapse: ReturnType, dataSetId: bigint): Promise { + for (let attempt = 0; attempt < 30; attempt++) { + const dataSet = await getPdpDataSet(synapse.client, { dataSetId }) + if (dataSet != null) { + const rail = await getRail(synapse.client, { railId: dataSet.pdpRailId }) + if (rail.endEpoch > 0n) return rail.endEpoch + } + await delay(1000) + } + throw new Error(`Data set ${dataSetId} payment rail did not become terminated`) +} + +async function main(): Promise { + const environment = resolveEnvironment({ defaultUserIndex: 0, requireFiles: true }) + if (environment.filePaths.length !== 1) throw new Error('termination-controls.ts accepts exactly one file path') + const [filePath] = environment.filePaths + const synapse = createSynapse(environment) + await prepareAccount(synapse, (await fileSize(filePath)) * 2n) + + const { result } = await uploadFile(synapse, filePath, freshMetadata('termination-controls'), 2) + assert.equal(result.copies.length, 2, 'Expected two independent data sets') + const [relayedCopy, directCopy] = result.copies + const priceList = await getPriceList(synapse.client) + + const beforeRelayed = await readAccountState(synapse) + const relayed = await synapse.storage.terminateService({ + dataSetId: relayedCopy.dataSetId, + onSubmitted: (hash) => console.log(`SP-relayed termination submitted: ${hash}`), + }) + assert.equal(relayed.dataSetId, relayedCopy.dataSetId, 'Relayed termination returned the wrong data set') + assert(relayed.endEpoch > 0n, 'Relayed termination returned an empty end epoch') + const relayedRailEndEpoch = await waitForTerminatedRail(synapse, relayedCopy.dataSetId) + assert.equal(relayedRailEndEpoch, relayed.endEpoch, 'Relayed termination result differs from the rail end epoch') + const afterRelayed = await readAccountState(synapse) + assert( + beforeRelayed.funds - afterRelayed.funds >= priceList.fees.terminateFee, + 'SP-relayed termination did not charge at least the configured termination fee' + ) + + const beforeDirect = await readAccountState(synapse) + const direct = await synapse.storage.terminateService({ + dataSetId: directCopy.dataSetId, + skipProvider: true, + onSubmitted: (hash) => console.log(`Direct termination submitted: ${hash}`), + }) + assert.equal(direct.dataSetId, directCopy.dataSetId, 'Direct termination returned the wrong data set') + assert(direct.txHash != null, 'Direct termination did not submit an on-chain transaction') + assert(direct.endEpoch > 0n, 'Direct termination returned an empty end epoch') + const directRailEndEpoch = await waitForTerminatedRail(synapse, directCopy.dataSetId) + assert.equal(directRailEndEpoch, direct.endEpoch, 'Direct termination result differs from the rail end epoch') + const afterDirect = await readAccountState(synapse) + assert.equal( + beforeDirect.funds - afterDirect.funds, + 0n, + 'Direct termination charged the SP-mediated termination fee' + ) + console.log('=== SUCCESS: relayed and direct termination controls observed; no cleanup wait performed ===') +} + +main().catch((error: unknown) => { + console.error(error) + process.exitCode = 1 +}) diff --git a/scenarios/synapse_runtime.py b/scenarios/synapse_runtime.py index a01caaf..49f6ab2 100755 --- a/scenarios/synapse_runtime.py +++ b/scenarios/synapse_runtime.py @@ -14,7 +14,10 @@ from scenarios.dependencies import component from scenarios.helpers import fail, info, ok, run_cmd -STATE_FORK_ERROR = "refusing explicit call due to state fork at epoch" +TRANSIENT_CHAIN_ERRORS = ( + "refusing explicit call due to state fork at epoch", + "requested epoch was a null round", +) UPLOAD_RETRY_DELAYS_SECS = (5, 10, 15, 30) @@ -153,7 +156,7 @@ def run_node_script( env: dict | None = None, timeout: int | None = None, ) -> None: - """Run a prepared scenario entrypoint with retries for transient state forks.""" + """Run a prepared scenario entrypoint with retries for transient chain errors.""" script = runtime.work_dir / script_name if not script.is_file(): raise RuntimeError(f"Synapse scenario entrypoint not found: {script}") @@ -179,12 +182,15 @@ def run_node_script( info(details) ok(label) return - if STATE_FORK_ERROR not in details or attempt == max_attempts: + if ( + not any(error in details for error in TRANSIENT_CHAIN_ERRORS) + or attempt == max_attempts + ): fail(f"{label} (exit={result.returncode}) {details}") delay = UPLOAD_RETRY_DELAYS_SECS[attempt - 1] info( - f"{label}: Lotus refused eth_call while crossing a state fork; " + f"{label}: Lotus returned a transient chain RPC error; " f"retrying in {delay}s (attempt {attempt}/{max_attempts})" ) time.sleep(delay) diff --git a/scenarios/test_bulk_add.py b/scenarios/test_bulk_add.py new file mode 100644 index 0000000..6268f75 --- /dev/null +++ b/scenarios/test_bulk_add.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Exercise the optional many-piece and lockup-replenishment path.""" + +import os +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from scenarios.helpers import assert_eq, assert_ok, info, write_random_file +from scenarios.synapse_runtime import prepare_synapse_runtime, run_node_script + +BOOTSTRAP_SIZE = 64 * 1024 + + +def run(): + assert_ok("command -v node", "node is installed") + with tempfile.TemporaryDirectory(prefix="bulk-add-") as tmp: + runtime = prepare_synapse_runtime(Path(tmp)) + fixture = runtime.work_dir / "bulk-bootstrap.bin" + write_random_file(fixture, BOOTSTRAP_SIZE, seed=12740) + assert_eq( + fixture.stat().st_size, BOOTSTRAP_SIZE, "bulk-add bootstrap fixture created" + ) + info("Running optional 40-piece add and lockup-replenishment checks") + run_node_script( + runtime, + "bulk-add.ts", + "bulk add scenario", + args=[str(fixture)], + env={"NETWORK": "devnet"}, + ) + + +if __name__ == "__main__": + run() diff --git a/scenarios/test_negative_permissions.py b/scenarios/test_negative_permissions.py new file mode 100644 index 0000000..920b761 --- /dev/null +++ b/scenarios/test_negative_permissions.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +"""Verify rejected service-provider requests cannot mutate a live data set.""" + +import os +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from scenarios.helpers import assert_eq, assert_ok, info, write_random_file +from scenarios.synapse_runtime import prepare_synapse_runtime, run_node_script + +FIXTURE_SIZE = 4 * 1024 + + +def run(): + assert_ok("command -v node", "node is installed") + with tempfile.TemporaryDirectory(prefix="negative-permissions-") as tmp: + runtime = prepare_synapse_runtime(Path(tmp)) + fixture = runtime.work_dir / "negative-permissions.bin" + write_random_file(fixture, FIXTURE_SIZE, seed=127) + assert_eq( + fixture.stat().st_size, FIXTURE_SIZE, "negative permissions fixture created" + ) + info( + "Running permission and malformed-request checks against an isolated data set" + ) + run_node_script( + runtime, + "negative-permissions.ts", + "negative permissions scenario", + args=[str(fixture)], + env={"NETWORK": "devnet"}, + ) + + +if __name__ == "__main__": + run() diff --git a/scenarios/test_termination_controls.py b/scenarios/test_termination_controls.py new file mode 100644 index 0000000..5a76e6e --- /dev/null +++ b/scenarios/test_termination_controls.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Exercise optional relayed and direct data-set termination controls.""" + +import os +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from scenarios.helpers import assert_eq, assert_ok, info, write_random_file +from scenarios.synapse_runtime import prepare_synapse_runtime, run_node_script + +FIXTURE_SIZE = 4 * 1024 + + +def run(): + assert_ok("command -v node", "node is installed") + with tempfile.TemporaryDirectory(prefix="termination-controls-") as tmp: + runtime = prepare_synapse_runtime(Path(tmp)) + fixture = runtime.work_dir / "termination-controls.bin" + write_random_file(fixture, FIXTURE_SIZE, seed=127900) + assert_eq( + fixture.stat().st_size, FIXTURE_SIZE, "termination controls fixture created" + ) + info("Running optional relayed and direct termination checks") + run_node_script( + runtime, + "termination-controls.ts", + "termination controls scenario", + args=[str(fixture)], + env={"NETWORK": "devnet"}, + ) + + +if __name__ == "__main__": + run() diff --git a/scripts/tests/test_scenario_dependencies.py b/scripts/tests/test_scenario_dependencies.py index e424c83..d44bf06 100644 --- a/scripts/tests/test_scenario_dependencies.py +++ b/scripts/tests/test_scenario_dependencies.py @@ -247,6 +247,36 @@ def test_run_node_script_retries_state_fork_error(self, run, _info, ok, sleep): sleep.assert_called_once_with(5) ok.assert_called_once_with("run smoke") + @patch("scenarios.synapse_runtime.time.sleep") + @patch("scenarios.synapse_runtime.ok") + @patch("scenarios.synapse_runtime.info") + @patch("scenarios.synapse_runtime.subprocess.run") + def test_run_node_script_retries_null_round_error(self, run, _info, ok, sleep): + run.side_effect = [ + subprocess.CompletedProcess( + ["node", "smoke.ts"], + 1, + stdout='Request body: {"method":"eth_getBlockByNumber"}', + stderr="Details: requested epoch was a null round (215)", + ), + subprocess.CompletedProcess( + ["node", "smoke.ts"], 0, stdout="done\n", stderr="" + ), + ] + + with tempfile.TemporaryDirectory() as directory: + work_dir = Path(directory) + (work_dir / "smoke.ts").touch() + run_node_script( + SynapseRuntime(work_dir, "npm", "npm:@filoz/synapse-sdk@1.1.1"), + "smoke.ts", + "run smoke", + ) + + self.assertEqual(run.call_count, 2) + sleep.assert_called_once_with(5) + ok.assert_called_once_with("run smoke") + @patch("scenarios.test_multi_copy_upload.run_cmd", return_value=True) @patch( "scenarios.test_multi_copy_upload.component", diff --git a/scripts/tests/test_scenario_runner.py b/scripts/tests/test_scenario_runner.py new file mode 100644 index 0000000..fca64f7 --- /dev/null +++ b/scripts/tests/test_scenario_runner.py @@ -0,0 +1,57 @@ +"""Unit tests for scenario selection without starting a devnet.""" + +import sys +import unittest +from unittest.mock import patch + +from scenarios.report import _format_skipped_optional_tests +from scenarios.run import ORDER, _parse_args, select_scenarios + + +class ScenarioSelectionTests(unittest.TestCase): + def test_core_selection_excludes_optional_scenarios(self): + selected, skipped = select_scenarios(False) + + self.assertEqual( + [entry[0] for entry in skipped], + ["test_bulk_add", "test_termination_controls"], + ) + self.assertEqual( + [entry[0] for entry in selected], [entry[0] for entry in ORDER[:-2]] + ) + self.assertTrue(all(not entry[2] for entry in selected)) + + def test_optional_selection_runs_everything_in_order(self): + selected, skipped = select_scenarios(True) + + self.assertEqual(selected, ORDER) + self.assertEqual(skipped, []) + + def test_include_optional_cli_flag_selects_every_scenario(self): + with patch.object(sys, "argv", ["run.py", "--include-optional"]): + args = _parse_args() + + selected, skipped = select_scenarios(args.include_optional) + self.assertTrue(args.include_optional) + self.assertEqual(selected, ORDER) + self.assertEqual(skipped, []) + + def test_selection_preserves_custom_order_and_report_data(self): + order = [("core", 1, False), ("extended", 2, True)] + + selected, skipped = select_scenarios(False, order) + + self.assertEqual(selected, [("core", 1, False)]) + self.assertEqual(skipped, [("extended", 2, True)]) + + def test_skipped_scenarios_are_rendered_for_the_report(self): + _, skipped = select_scenarios(False) + + rendered = _format_skipped_optional_tests([entry[0] for entry in skipped]) + + self.assertIn("`test_bulk_add`", rendered) + self.assertIn("`test_termination_controls`", rendered) + + +if __name__ == "__main__": + unittest.main()