diff --git a/circle.yml b/circle.yml index 58e49b6253..e02d167ee5 100644 --- a/circle.yml +++ b/circle.yml @@ -177,14 +177,14 @@ commands: working_directory: ~/build command: > LLVM_PROFILE_FILE=state_tests.profraw - bin/evmone-statetest <> ~/spec-tests/fixtures/state_tests + bin/evmone test <> ~/spec-tests/fixtures/state_tests - run: name: "Execution spec tests (<>, blockchain_tests)" # Tests for in-development EVM revision currently passing. working_directory: ~/build command: > LLVM_PROFILE_FILE=blockchain_tests.profraw - bin/evmone-blockchaintest <> ~/spec-tests/fixtures/blockchain_tests + bin/evmone test <> ~/spec-tests/fixtures/blockchain_tests configure: description: "Configure" @@ -309,7 +309,7 @@ commands: default: "" binaries: type: string - default: evmone evmone-unittests evmone-statetest evmone-blockchaintest + default: evmone evmone-unittests steps: - run: name: "Collect coverage data (clang, <>)" @@ -472,13 +472,13 @@ jobs: - collect_coverage_clang: flags: eest-develop ignore_filename_regex: lib/evmone/(advanced|cpu_check|eof|lru_cache|tracing|vm)|test/(experimental|unittests) - binaries: evmone-statetest evmone-blockchaintest + binaries: evmone - run_execution_spec_tests: release: tests@v20.0.2 - collect_coverage_clang: flags: eest-stable ignore_filename_regex: lib/evmone/(advanced|cpu_check|eof|lru_cache|tracing|vm)|test/(experimental|unittests) - binaries: evmone-statetest evmone-blockchaintest + binaries: evmone ethereum-tests: executor: linux-clang-latest @@ -509,7 +509,7 @@ jobs: working_directory: ~/build command: > LLVM_PROFILE_FILE=blockchain_tests_valid.profraw - bin/evmone-blockchaintest + bin/evmone test ~/tests/BlockchainTests/ValidBlocks ~/tests/LegacyTests/Cancun/BlockchainTests/ValidBlocks - run: @@ -517,7 +517,7 @@ jobs: working_directory: ~/build command: > LLVM_PROFILE_FILE=blockchain_tests_invalid.profraw - bin/evmone-blockchaintest + bin/evmone test --ignore bc4895-withdrawals/shanghaiWithoutWithdrawalsRLP.json --ignore bcInvalidHeaderTest --ignore bcUncleHeaderValidity/gasLimitTooLowExactBound.json @@ -539,7 +539,7 @@ jobs: name: "Execution spec tests (state_tests)" working_directory: ~/build command: > - bin/evmone-statetest ~/spec-tests/fixtures/state_tests + bin/evmone test ~/spec-tests/fixtures/state_tests - collect_coverage_clang: flags: eest-libsecp256k1 @@ -559,7 +559,7 @@ jobs: name: "Execution spec tests (state_tests)" working_directory: ~/build command: > - bin/evmone-statetest ~/spec-tests/fixtures/state_tests + bin/evmone test ~/spec-tests/fixtures/state_tests - collect_coverage_clang: flags: eest-develop-gmp diff --git a/codecov.yml b/codecov.yml index 10161ea7b2..c223503518 100644 --- a/codecov.yml +++ b/codecov.yml @@ -19,10 +19,8 @@ component_management: paths: - tools/** - test/bench/** - - test/blockchaintest/** - test/eofparse/** - test/eoftest/** - - test/statetest/** - test/utils/** - component_id: tests name: tests diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index b8105f8238..c56bf1526f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -53,15 +53,13 @@ include(CableBuildInfo) cable_add_buildinfo_library(PROJECT_NAME evmone) add_subdirectory(bench) -add_subdirectory(blockchaintest) add_subdirectory(experimental) add_subdirectory(integration) add_subdirectory(internal_benchmarks) add_subdirectory(precompiles_bench) -add_subdirectory(statetest) add_subdirectory(unittests) -set(targets evmone-bench evmone-bench-internal evmone-blockchaintest evmone-precompiles-bench evmone-state evmone-statetest evmone-unittests) +set(targets evmone-bench evmone-bench-internal evmone-precompiles-bench evmone-state evmone-unittests) if(EVMONE_FUZZING) add_subdirectory(fuzzer) diff --git a/test/blockchaintest/.clang-tidy b/test/blockchaintest/.clang-tidy deleted file mode 100644 index efc628c8e9..0000000000 --- a/test/blockchaintest/.clang-tidy +++ /dev/null @@ -1,3 +0,0 @@ -InheritParentConfig: true -Checks: > - -clang-analyzer-cplusplus.NewDeleteLeaks diff --git a/test/blockchaintest/CMakeLists.txt b/test/blockchaintest/CMakeLists.txt deleted file mode 100644 index 79b42d9085..0000000000 --- a/test/blockchaintest/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -# evmone: Fast Ethereum Virtual Machine implementation -# Copyright 2023 The evmone Authors. -# SPDX-License-Identifier: Apache-2.0 - -add_executable(evmone-blockchaintest) -target_link_libraries(evmone-blockchaintest PRIVATE evmone::testutils evmone evmone-buildinfo CLI11::CLI11) -target_sources( - evmone-blockchaintest PRIVATE - blockchaintest.cpp -) diff --git a/test/blockchaintest/blockchaintest.cpp b/test/blockchaintest/blockchaintest.cpp deleted file mode 100644 index 3e736f5305..0000000000 --- a/test/blockchaintest/blockchaintest.cpp +++ /dev/null @@ -1,94 +0,0 @@ -// evmone: Fast Ethereum Virtual Machine implementation -// Copyright 2023 The evmone Authors. -// SPDX-License-Identifier: Apache-2.0 - -#include -#include -#include -#include -#include -#include -#include - -namespace fs = std::filesystem; -using evmone::test::TestCase; - -namespace -{ -/// Adds to @p cases one test per fixture file under @p root, which is that file itself when it -/// is not a directory. -void collect_tests(std::vector& cases, const fs::path& root, - std::span ignored, evmc::VM& vm) -{ - // A file named directly is its own collection; the ignored paths are relative to the - // directory holding it, as they are to a directory named directly. - const auto is_dir = is_directory(root); - auto files = is_dir ? evmone::test::collect_test_files(root) : std::vector{root}; - evmone::test::ignore_test_files(files, is_dir ? root : root.parent_path(), ignored); - - cases.reserve(cases.size() + files.size()); - for (const auto& path : files) - { - // Loaded when the test runs: loading a whole tree up front costs far more. A - // load which throws over an unsupported fixture reaches the driver, which skips. - cases.push_back({path.string(), [path, &vm](evmone::test::TestReport& report) { - std::ifstream f{path}; - for (const auto& test : evmone::test::load_blockchain_tests(f)) - evmone::test::run_blockchain_test(test, vm, report); - }}); - } -} -} // namespace - - -int main(int argc, char* argv[]) -{ - try - { - CLI::App app{"evmone blockchain test runner"}; - - app.set_version_flag("--version", "evmone-blockchaintest " EVMONE_VERSION); - - std::vector paths; - app.add_option("path", paths, - "Path to a test file or a directory of them. Under a directory every .json " - "file except index.json is one test; a file named directly is one test.") - ->required() - ->check(CLI::ExistingPath); - - std::vector ignored; - app.add_option("--ignore", ignored, - "Path, relative to a given path, not to collect tests from. May be given more " - "than once. Whole path components are matched, so --ignore bc4895 keeps " - "bc4895-withdrawals.") - // Without this the option is variadic and swallows the positional paths after it. - ->allow_extra_args(false); - - bool collect_only = false; - app.add_flag("--collect-only", collect_only, - "List the path of each collected test, one per line, and exit."); - - bool trace_flag = false; - app.add_flag("--trace", trace_flag, "Enable EVM tracing"); - - CLI11_PARSE(app, argc, argv); - - evmc::VM vm{evmc_create_evmone()}; - - if (trace_flag) - vm.set_option("trace", "1"); - - std::vector cases; - for (const auto& p : paths) - collect_tests(cases, p, ignored, vm); - - const evmone::test::RunOptions options{ - .collect_only = collect_only, .progress = !trace_flag}; - return evmone::test::run_tests(cases, std::cout, options); - } - catch (const std::exception& ex) - { - std::cerr << ex.what() << "\n"; - return -1; - } -} diff --git a/test/integration/evmone-cli/test/CMakeLists.txt b/test/integration/evmone-cli/test/CMakeLists.txt index 1ed8c4a87b..8b3bcc4b40 100644 --- a/test/integration/evmone-cli/test/CMakeLists.txt +++ b/test/integration/evmone-cli/test/CMakeLists.txt @@ -6,38 +6,78 @@ set(PREFIX ${PREFIX}/test) -# A file holding a case which is not a fixture at all, beside one which runs. -add_test(NAME ${PREFIX}/fixture_fault COMMAND evmone-statetest - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault) -set_tests_properties(${PREFIX}/fixture_fault PROPERTIES WILL_FAIL TRUE) - -# WILL_FAIL above accepts any non-zero exit, and a fault downgraded to a skip exits non-zero too, -# so the counts are what distinguish the two. -add_test(NAME ${PREFIX}/fixture_fault_is_not_a_skip COMMAND evmone-statetest - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault) -set_tests_properties( - ${PREFIX}/fixture_fault_is_not_a_skip PROPERTIES PASS_REGULAR_EXPRESSION "1 failed, 0 passed") - -# Selecting only the case which is not a fixture must still fault. -add_test(NAME ${PREFIX}/fixture_fault_survives_filter COMMAND evmone-statetest - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault -k b_not_a_fixture) -set_tests_properties( - ${PREFIX}/fixture_fault_survives_filter PROPERTIES - PASS_REGULAR_EXPRESSION "1 failed, 0 passed") - -# A case whose load throws takes the rest of the file with it: the whole file is loaded before -# any of it runs, so the case after it is never reached and the failure is named after the file -# rather than the case it came from. FAILED pins that as a failure: a PASS_REGULAR_EXPRESSION -# makes CTest ignore the exit code, and a fault downgraded to a skip names the file too. -add_test(NAME ${PREFIX}/case_after_exception COMMAND evmone-statetest - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_cases) -set_tests_properties( - ${PREFIX}/case_after_exception PROPERTIES - PASS_REGULAR_EXPRESSION - "collected 1 test.*FAILED[^\n]*case_after_exception\\.json[^\n]*exception" - FAIL_REGULAR_EXPRESSION "b_wrong_state_root" -) +set(DATA ${CMAKE_CURRENT_SOURCE_DIR}) +# One run takes both formats: eip7778_block_gas.json declares its format, the other two are +# recognised by shape. +add_test(NAME ${PREFIX}/test_mixed_formats COMMAND evmone-cli test + ${DATA}/statetest/tests1/SuiteA/test1.json ${DATA}/blockchaintest/eip7778_block_gas.json + ${DATA}/blockchaintest/unrecovered_sender_blob_gas.json) +set_tests_properties( + ${PREFIX}/test_mixed_formats PROPERTIES PASS_REGULAR_EXPRESSION "= 3 passed in") + +# A file recognised as "not a test" is skipped. +add_test(NAME ${PREFIX}/test_not_a_test COMMAND evmone-cli test + ${DATA}/statetest/tests1/SuiteA/index.json) +set_tests_properties( + ${PREFIX}/test_not_a_test PROPERTIES PASS_REGULAR_EXPRESSION + "SKIPPED[^\n]*index\\.json - not a test.*0 passed, 1 skipped in") + +# Check exit code for: passed, failed, skipped (PASS_REGULAR_EXPRESSION ignores exit code). +add_test(NAME ${PREFIX}/test_exit_ok COMMAND evmone-cli test ${DATA}/testcmd) +add_test(NAME ${PREFIX}/test_exit_fail COMMAND evmone-cli test ${DATA}/testcmd_fault) +set_tests_properties(${PREFIX}/test_exit_fail PROPERTIES WILL_FAIL TRUE) +add_test(NAME ${PREFIX}/test_exit_skipped COMMAND evmone-cli test ${DATA}/testcmd_skipped) +set_tests_properties(${PREFIX}/test_exit_skipped PROPERTIES WILL_FAIL TRUE) + +# A file passes on the fixtures it ran; the two it declined are named with their reasons. +add_test(NAME ${PREFIX}/test_dir_declined COMMAND evmone-cli test ${DATA}/testcmd) +set_tests_properties( + ${PREFIX}/test_dir_declined PROPERTIES PASS_REGULAR_EXPRESSION + "SKIPPED[^\n]*::a_bad_rlp - tests with invalidly rlp-encoded blocks[^\n]*\nSKIPPED[^\n]*::c_engine - unsupported fixture format.*= 1 passed in") + +# A selected "not a test" fails even when the runnable fixtures beside it are filtered out. +add_test(NAME ${PREFIX}/test_dir_fault COMMAND evmone-cli test + ${DATA}/testcmd_fault -k b_not_a_test) +set_tests_properties( + ${PREFIX}/test_dir_fault PROPERTIES PASS_REGULAR_EXPRESSION "1 failed, 0 passed") + +# A fixture whose load throws fails only itself; the one after it still runs. +add_test(NAME ${PREFIX}/test_after_exception COMMAND evmone-cli test ${DATA}/testcmd_fault) +set_tests_properties( + ${PREFIX}/test_after_exception PROPERTIES PASS_REGULAR_EXPRESSION + "collected 1 file\n.*c_load_error:\n exception.*d_bad_root:.*state root.*1 failed, 0 passed in") + +# A file is one test, so a file with nothing to run counts once as skipped however many +# fixtures it declined: all_unsupported declines twice and both are named. +add_test(NAME ${PREFIX}/test_dir_skipped COMMAND evmone-cli test ${DATA}/testcmd_skipped) +set_tests_properties( + ${PREFIX}/test_dir_skipped PROPERTIES PASS_REGULAR_EXPRESSION + "all_unsupported\\.json::a_engine - unsupported fixture format: \"blockchain_test_engine\"\nSKIPPED[^\n]*::b_engine_x - .*empty\\.json - not a test.*pre_alloc\\.json - not a test.*0 passed, 5 skipped in") + +# The --histogram output reaches the report, and the progress row keeps out of its way: +# a run without it ends every line with the percentage done. +add_test(NAME ${PREFIX}/test_histogram COMMAND evmone-cli test --histogram + ${DATA}/statetest/tests1/SuiteA/test1.json) +set_tests_properties( + ${PREFIX}/test_histogram PROPERTIES + PASS_REGULAR_EXPRESSION "--- # HISTOGRAM.*= 1 passed in" + FAIL_REGULAR_EXPRESSION "\\[100%\\]") + +# The --trace output likewise. +add_test(NAME ${PREFIX}/test_trace COMMAND evmone-cli test --trace + ${DATA}/statetest/tests1/SuiteA/test1.json) +set_tests_properties( + ${PREFIX}/test_trace PROPERTIES + PASS_REGULAR_EXPRESSION "\\{\"pc\":0,[^\n]*\"opName\":\"PUSH1\"\\}.*= 1 passed in" + FAIL_REGULAR_EXPRESSION "\\[100%\\]") + +# A named file is loaded whatever its extension. +add_test(NAME ${PREFIX}/test_file_unparsed COMMAND evmone-cli test + ${DATA}/statetest/tests1/SuiteA/notes.txt) +set_tests_properties( + ${PREFIX}/test_file_unparsed PROPERTIES PASS_REGULAR_EXPRESSION + "FAILED[^\n]*notes\\.txt - exception[^\n]*parse error.*1 failed, 0 passed in") get_directory_property(ALL_TESTS TESTS) set_tests_properties(${ALL_TESTS} PROPERTIES ENVIRONMENT LLVM_PROFILE_FILE=${CMAKE_BINARY_DIR}/integration-%p.profraw) diff --git a/test/integration/evmone-cli/test/blockchaintest/CMakeLists.txt b/test/integration/evmone-cli/test/blockchaintest/CMakeLists.txt index f378215cf4..1035452b7a 100644 --- a/test/integration/evmone-cli/test/blockchaintest/CMakeLists.txt +++ b/test/integration/evmone-cli/test/blockchaintest/CMakeLists.txt @@ -2,25 +2,25 @@ # Copyright 2022 The evmone Authors. # SPDX-License-Identifier: Apache-2.0 -# Integration tests for evmone-blockchaintest. +# Integration tests for `evmone test` over blockchain test fixtures. set(PREFIX ${PREFIX}/blockchaintest) set(TESTS1 ${CMAKE_CURRENT_SOURCE_DIR}) add_test( NAME ${PREFIX}/json_test - COMMAND evmone-blockchaintest ${TESTS1}/test.json + COMMAND evmone-cli test ${TESTS1}/test.json ) set_tests_properties( ${PREFIX}/json_test PROPERTIES # Both fixtures of the file run and both fail; the file itself counts once. - PASS_REGULAR_EXPRESSION "-call\\]:.*-callcode\\]:.*1 failed, 0 passed" + PASS_REGULAR_EXPRESSION "-call\\] - canonical.*-callcode\\] - canonical.*1 failed, 0 passed" ) # Exercise block-level gas accounting (EIP-7778), with tracing on so that flag is covered too. add_test( NAME ${PREFIX}/eip7778_block_gas - COMMAND evmone-blockchaintest ${TESTS1}/eip7778_block_gas.json --trace + COMMAND evmone-cli test ${TESTS1}/eip7778_block_gas.json --trace ) set_tests_properties( ${PREFIX}/eip7778_block_gas PROPERTIES @@ -31,18 +31,18 @@ set_tests_properties( # A block invalid by its signature must not be judged by another check. add_test( NAME ${PREFIX}/unrecovered_sender_blob_gas - COMMAND evmone-blockchaintest ${TESTS1}/unrecovered_sender_blob_gas.json + COMMAND evmone-cli test ${TESTS1}/unrecovered_sender_blob_gas.json ) add_test( NAME ${PREFIX}/unrecovered_sender_deposit_log - COMMAND evmone-blockchaintest ${TESTS1}/unrecovered_sender_deposit_log.json + COMMAND evmone-cli test ${TESTS1}/unrecovered_sender_deposit_log.json ) # A block with expectException but no rlp_decoded is a shape the loader does not support, so the # only test in this file is skipped and the run verifies nothing. add_test( NAME ${PREFIX}/unsupported_rlp - COMMAND evmone-blockchaintest ${TESTS1}/unsupported_rlp.json + COMMAND evmone-cli test ${TESTS1}/unsupported_rlp.json ) set_tests_properties( ${PREFIX}/unsupported_rlp PROPERTIES @@ -53,7 +53,7 @@ set_tests_properties( # A PASS_REGULAR_EXPRESSION makes CTest ignore the exit code, so run it again for that alone. add_test( NAME ${PREFIX}/unsupported_rlp_exit_code - COMMAND evmone-blockchaintest ${TESTS1}/unsupported_rlp.json + COMMAND evmone-cli test ${TESTS1}/unsupported_rlp.json ) set_tests_properties(${PREFIX}/unsupported_rlp_exit_code PROPERTIES WILL_FAIL TRUE) @@ -61,18 +61,18 @@ set_tests_properties(${PREFIX}/unsupported_rlp_exit_code PROPERTIES WILL_FAIL TR # the unsupported one rather than failing on it. add_test( NAME ${PREFIX}/directory - COMMAND evmone-blockchaintest ${TESTS1} + COMMAND evmone-cli test ${TESTS1} ) set_tests_properties( ${PREFIX}/directory PROPERTIES - PASS_REGULAR_EXPRESSION "SKIPPED [^\n]*unsupported_rlp\\.json - tests with invalidly" + PASS_REGULAR_EXPRESSION "SKIPPED [^\n]*unsupported_rlp\\.json::[^\n]* - tests with invalidly" ) # A test which gives up gives up only itself. Collected from a directory the whole file is one # test, so the test after the one which stopped has to report too. add_test( NAME ${PREFIX}/stops_at_one_test - COMMAND evmone-blockchaintest ${TESTS1} + COMMAND evmone-cli test ${TESTS1} ) set_tests_properties( ${PREFIX}/stops_at_one_test PROPERTIES @@ -86,7 +86,7 @@ set_tests_properties( # list. Over a directory a test is listed as its file. add_test( NAME ${PREFIX}/collect_only_directory - COMMAND evmone-blockchaintest ${TESTS1} --collect-only --ignore unsupported_rlp.json + COMMAND evmone-cli test ${TESTS1} --collect-only --ignore unsupported_rlp.json ) set_tests_properties( ${PREFIX}/collect_only_directory PROPERTIES @@ -94,36 +94,5 @@ set_tests_properties( FAIL_REGULAR_EXPRESSION "unsupported_rlp" ) -# A file named directly is listed as itself, once, whatever it holds. -add_test( - NAME ${PREFIX}/collect_only_file - COMMAND evmone-blockchaintest ${TESTS1}/test.json --collect-only -) -set_tests_properties( - ${PREFIX}/collect_only_file PROPERTIES - PASS_REGULAR_EXPRESSION "^[^\n]*test\\.json\n$" -) - -# Collection reads no file, so one which cannot be parsed is listed like any other. It fails -# when it runs, which is the only time anything reads it. -add_test( - NAME ${PREFIX}/collect_only_unloadable - COMMAND evmone-blockchaintest ${TESTS1}/not_json.txt --collect-only -) -set_tests_properties( - ${PREFIX}/collect_only_unloadable PROPERTIES - PASS_REGULAR_EXPRESSION "^[^\n]*not_json\\.txt\n$" -) - -# Pointing at a not-json file produces a failure during test execution. -add_test( - NAME ${PREFIX}/run_unloadable - COMMAND evmone-blockchaintest ${TESTS1}/not_json.txt -) -set_tests_properties( - ${PREFIX}/run_unloadable PROPERTIES - PASS_REGULAR_EXPRESSION "1 failed, 0 passed" -) - get_directory_property(ALL_TESTS TESTS) set_tests_properties(${ALL_TESTS} PROPERTIES ENVIRONMENT LLVM_PROFILE_FILE=${CMAKE_BINARY_DIR}/integration-%p.profraw) diff --git a/test/integration/evmone-cli/test/blockchaintest/eip7778_block_gas.json b/test/integration/evmone-cli/test/blockchaintest/eip7778_block_gas.json index 326c57ef0b..a9642ccc1f 100644 --- a/test/integration/evmone-cli/test/blockchaintest/eip7778_block_gas.json +++ b/test/integration/evmone-cli/test/blockchaintest/eip7778_block_gas.json @@ -130,6 +130,9 @@ }, "genesisRLP": "0x", "sealEngine": "NoProof", - "_info": {} + "_info": { + "fixture-format": "blockchain_test", + "comment": "Declared as EEST does, so a run takes the declared-format path rather than the shape's." + } } } diff --git a/test/integration/evmone-cli/test/blockchaintest/not_json.txt b/test/integration/evmone-cli/test/blockchaintest/not_json.txt deleted file mode 100644 index 2fb687897c..0000000000 --- a/test/integration/evmone-cli/test/blockchaintest/not_json.txt +++ /dev/null @@ -1 +0,0 @@ -Not JSON, so the loader cannot read this as a test file. diff --git a/test/integration/evmone-cli/test/statetest/CMakeLists.txt b/test/integration/evmone-cli/test/statetest/CMakeLists.txt index 6fd23e62a0..0df13c50b8 100644 --- a/test/integration/evmone-cli/test/statetest/CMakeLists.txt +++ b/test/integration/evmone-cli/test/statetest/CMakeLists.txt @@ -2,7 +2,7 @@ # Copyright 2022 The evmone Authors. # SPDX-License-Identifier: Apache-2.0 -# Integration tests for evmone-statetest. +# Integration tests for `evmone test` over state test fixtures. set(PREFIX ${PREFIX}/statetest) set(TESTS1 ${CMAKE_CURRENT_SOURCE_DIR}/tests1) @@ -11,7 +11,7 @@ set(TESTS_FILTER ${CMAKE_CURRENT_SOURCE_DIR}/filter) add_test( NAME ${PREFIX}/no_arguments - COMMAND evmone-statetest + COMMAND evmone-cli test ) set_tests_properties( ${PREFIX}/no_arguments PROPERTIES @@ -23,7 +23,7 @@ set_tests_properties( # show up in the listing. add_test( NAME ${PREFIX}/tests1_list - COMMAND evmone-statetest ${TESTS1} --collect-only + COMMAND evmone-cli test ${TESTS1} --collect-only ) set_tests_properties( ${PREFIX}/tests1_list PROPERTIES @@ -33,7 +33,7 @@ set_tests_properties( # A file named directly is listed as itself, whatever it holds. add_test( NAME ${PREFIX}/single_file_list - COMMAND evmone-statetest ${TESTS1}/SuiteA/test2_multi.json --collect-only + COMMAND evmone-cli test ${TESTS1}/SuiteA/test2_multi.json --collect-only ) set_tests_properties( ${PREFIX}/single_file_list PROPERTIES @@ -41,10 +41,10 @@ set_tests_properties( ) # Several roots are collected in the order given, not regrouped by suite as gtest listed them. -# T.json holds no test case, but naming it is naming a test, so it is listed like any other. +# T.json holds no fixture, but naming it is naming a test, so it is listed like any other. add_test( NAME ${PREFIX}/multiple_args_list - COMMAND evmone-statetest ${TESTS1} ${TESTS2} ${TESTS1}/B/T.json ${TESTS1}/SuiteA --collect-only + COMMAND evmone-cli test ${TESTS1} ${TESTS2} ${TESTS1}/B/T.json ${TESTS1}/SuiteA --collect-only ) set_tests_properties( ${PREFIX}/multiple_args_list PROPERTIES @@ -53,7 +53,7 @@ set_tests_properties( add_test( NAME ${PREFIX}/invalid_path - COMMAND evmone-statetest invalid.json + COMMAND evmone-cli test invalid.json ) set_tests_properties( ${PREFIX}/invalid_path PROPERTIES @@ -62,7 +62,7 @@ set_tests_properties( add_test( NAME ${PREFIX}/multi_test - COMMAND evmone-statetest ${TESTS1}/SuiteA/test2_multi.json + COMMAND evmone-cli test ${TESTS1}/SuiteA/test2_multi.json ) set_tests_properties( ${PREFIX}/multi_test PROPERTIES @@ -72,7 +72,7 @@ set_tests_properties( add_test( NAME ${PREFIX}/trace - COMMAND evmone-statetest ${TESTS1}/SuiteA/test1.json --trace + COMMAND evmone-cli test ${TESTS1}/SuiteA/test1.json --trace ) set_tests_properties( ${PREFIX}/trace PROPERTIES @@ -84,7 +84,7 @@ set_tests_properties( add_test( NAME ${PREFIX}/filter - COMMAND evmone-statetest ${TESTS_FILTER}/one_failing_of_two.json -k passing_test_case + COMMAND evmone-cli test ${TESTS_FILTER}/one_failing_of_two.json -k passing_test_case ) set_tests_properties( ${PREFIX}/filter PROPERTIES @@ -100,7 +100,7 @@ set_tests_properties( # ran: forbidding the other name alone would hold just as well if the filter dropped every case. add_test( NAME ${PREFIX}/filter_directory - COMMAND evmone-statetest ${TESTS_FILTER} -k passing_test_case --trace-summary + COMMAND evmone-cli test ${TESTS_FILTER} -k passing_test_case --trace-summary ) set_tests_properties( ${PREFIX}/filter_directory PROPERTIES @@ -112,7 +112,7 @@ set_tests_properties( # that swallowed a positional path would leave the roots after it uncollected. add_test( NAME ${PREFIX}/ignore - COMMAND evmone-statetest --ignore B ${TESTS1} ${TESTS2} --collect-only + COMMAND evmone-cli test --ignore B ${TESTS1} ${TESTS2} --collect-only ) set_tests_properties( ${PREFIX}/ignore PROPERTIES @@ -123,7 +123,7 @@ set_tests_properties( # A file named directly is ignored the same way, by a path relative to that file. add_test( NAME ${PREFIX}/ignore_file - COMMAND evmone-statetest --ignore test1.json + COMMAND evmone-cli test --ignore test1.json ${TESTS1}/SuiteA/test1.json ${TESTS1}/SuiteA/test2_multi.json --collect-only ) set_tests_properties( @@ -135,30 +135,28 @@ set_tests_properties( # the driver unit tests pin the code itself. add_test( NAME ${PREFIX}/nothing_collected - COMMAND evmone-statetest ${TESTS1} --ignore B --ignore SuiteA + COMMAND evmone-cli test ${TESTS1} --ignore B --ignore SuiteA ) set_tests_properties(${PREFIX}/nothing_collected PROPERTIES WILL_FAIL TRUE) +# A filter which selects nothing verifies nothing either. The file passes when it runs, so +# the nonzero exit says the filter emptied it and not that something failed. +add_test( + NAME ${PREFIX}/filter_matches_nothing + COMMAND evmone-cli test ${TESTS1}/SuiteA/test1.json -k no_such_case +) +set_tests_properties(${PREFIX}/filter_matches_nothing PROPERTIES WILL_FAIL TRUE) + # Collection reads no file, so one which cannot be parsed is listed like any other. It fails # when it runs, which is the only time anything reads it. add_test( NAME ${PREFIX}/collect_only_unloadable - COMMAND evmone-statetest ${TESTS1}/SuiteA/notes.txt --collect-only + COMMAND evmone-cli test ${TESTS1}/SuiteA/notes.txt --collect-only ) set_tests_properties( ${PREFIX}/collect_only_unloadable PROPERTIES PASS_REGULAR_EXPRESSION "^[^\n]*notes\\.txt\n$" ) -# Pointing at a not-json file produces a failure during test execution. -add_test( - NAME ${PREFIX}/run_unloadable - COMMAND evmone-statetest ${TESTS1}/SuiteA/notes.txt -) -set_tests_properties( - ${PREFIX}/run_unloadable PROPERTIES - PASS_REGULAR_EXPRESSION "1 failed, 0 passed" -) - get_directory_property(ALL_TESTS TESTS) set_tests_properties(${ALL_TESTS} PROPERTIES ENVIRONMENT LLVM_PROFILE_FILE=${CMAKE_BINARY_DIR}/integration-%p.profraw) diff --git a/test/integration/evmone-cli/test/testcmd/declined.json b/test/integration/evmone-cli/test/testcmd/declined.json new file mode 100644 index 0000000000..e65f339135 --- /dev/null +++ b/test/integration/evmone-cli/test/testcmd/declined.json @@ -0,0 +1,99 @@ +{ + "a_bad_rlp": { + "_info": { + "comment": "Declines to load: expectException without rlp_decoded, which the loader refuses before reading anything after the blocks." + }, + "network": "Cancun", + "genesisBlockHeader": { + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "coinbase": "0x0000000000000000000000000000000000000000", + "stateRoot": "0x0000000000000000000000000000000000000000000000000000000000000000", + "transactionsTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "number": "0x00", + "gasLimit": "0x01000000", + "gasUsed": "0x00", + "timestamp": "0x00", + "extraData": "0x00", + "hash": "0x0000000000000000000000000000000000000000000000000000000000000001" + }, + "pre": {}, + "blocks": [ + { + "expectException": "TransactionException.INVALID_SIGNATURE_VRS", + "rlp": "0x" + } + ] + }, + "b_state": { + "_info": { + "comment": "A state test beside blockchain ones, which is the whole point of deciding the format per fixture." + }, + "env": { + "currentBaseFee": "0x0a", + "currentCoinbase": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "currentDifficulty": "0x020000", + "currentGasLimit": "0xff112233445566", + "currentNumber": "0x01", + "currentRandom": "0x0000000000000000000000000000000000000000000000000000000000020000", + "currentTimestamp": "0x03e8" + }, + "post": { + "London": [ + { + "hash": "0xe8010ce590f401c9d61fef8ab05bea9bcec24281b795e5868809bc4e515aa530", + "indexes": { + "data": 0, + "gas": 0, + "value": 0 + }, + "logs": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347" + } + ] + }, + "pre": { + "0x095e7baea6a6c7c4c2dfeb977efac326af552d87": { + "balance": "0x0de0b6b3a7640000", + "code": "0x600160010160005500", + "nonce": "0x00", + "storage": {} + }, + "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba": { + "balance": "0x00", + "code": "0x", + "nonce": "0x01", + "storage": {} + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0x0de0b6b3a7640000", + "code": "0x", + "nonce": "0x00", + "storage": {} + } + }, + "transaction": { + "data": [ + "0x" + ], + "gasLimit": [ + "0x061a80" + ], + "gasPrice": "0x0a", + "nonce": "0x00", + "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "to": "0x095e7baea6a6c7c4c2dfeb977efac326af552d87", + "value": [ + "0x0186a0" + ] + } + }, + "c_engine": { + "_info": { + "fixture-format": "blockchain_test_engine", + "comment": "Carries pre and blocks, so the shape alone would run it; the declared format is what decides, and this tool does not run that one." + }, + "pre": {}, + "blocks": [] + } +} diff --git a/test/integration/evmone-cli/test/testcmd_cases/case_after_exception.json b/test/integration/evmone-cli/test/testcmd_cases/case_after_exception.json deleted file mode 100644 index 0b8f8e37f7..0000000000 --- a/test/integration/evmone-cli/test/testcmd_cases/case_after_exception.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "a_load_error": { - "_info": { - "fixture-format": "state_test", - "comment": "A state test with no pre state, so loading it throws." - } - }, - "b_wrong_state_root": { - "_info": { - "comment": "Runs and fails on the state root. Only reported if the case before it did not abandon the file." - }, - "env": { - "currentBaseFee": "0x0a", - "currentCoinbase": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", - "currentDifficulty": "0x020000", - "currentGasLimit": "0xff112233445566", - "currentNumber": "0x01", - "currentRandom": "0x0000000000000000000000000000000000000000000000000000000000020000", - "currentTimestamp": "0x03e8" - }, - "post": { - "London": [ - { - "hash": "0x1111111111111111111111111111111111111111111111111111111111111111", - "indexes": { - "data": 0, - "gas": 0, - "value": 0 - }, - "logs": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347" - } - ] - }, - "pre": { - "0x095e7baea6a6c7c4c2dfeb977efac326af552d87": { - "balance": "0x0de0b6b3a7640000", - "code": "0x600160010160005500", - "nonce": "0x00", - "storage": {} - }, - "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba": { - "balance": "0x00", - "code": "0x", - "nonce": "0x01", - "storage": {} - }, - "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { - "balance": "0x0de0b6b3a7640000", - "code": "0x", - "nonce": "0x00", - "storage": {} - } - }, - "transaction": { - "data": [ - "0x" - ], - "gasLimit": [ - "0x061a80" - ], - "gasPrice": "0x0a", - "nonce": "0x00", - "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", - "to": "0x095e7baea6a6c7c4c2dfeb977efac326af552d87", - "value": [ - "0x0186a0" - ] - } - } -} diff --git a/test/integration/evmone-cli/test/testcmd_fault/faults.json b/test/integration/evmone-cli/test/testcmd_fault/faults.json new file mode 100644 index 0000000000..968e154118 --- /dev/null +++ b/test/integration/evmone-cli/test/testcmd_fault/faults.json @@ -0,0 +1,135 @@ +{ + "a_runs": { + "_info": {}, + "env": { + "currentBaseFee": "0x0a", + "currentCoinbase": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "currentDifficulty": "0x020000", + "currentGasLimit": "0xff112233445566", + "currentNumber": "0x01", + "currentRandom": "0x0000000000000000000000000000000000000000000000000000000000020000", + "currentTimestamp": "0x03e8" + }, + "post": { + "London": [ + { + "hash": "0xe8010ce590f401c9d61fef8ab05bea9bcec24281b795e5868809bc4e515aa530", + "indexes": { + "data": 0, + "gas": 0, + "value": 0 + }, + "logs": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347" + } + ] + }, + "pre": { + "0x095e7baea6a6c7c4c2dfeb977efac326af552d87": { + "balance": "0x0de0b6b3a7640000", + "code": "0x600160010160005500", + "nonce": "0x00", + "storage": {} + }, + "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba": { + "balance": "0x00", + "code": "0x", + "nonce": "0x01", + "storage": {} + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0x0de0b6b3a7640000", + "code": "0x", + "nonce": "0x00", + "storage": {} + } + }, + "transaction": { + "data": [ + "0x" + ], + "gasLimit": [ + "0x061a80" + ], + "gasPrice": "0x0a", + "nonce": "0x00", + "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "to": "0x095e7baea6a6c7c4c2dfeb977efac326af552d87", + "value": [ + "0x0186a0" + ] + } + }, + "b_not_a_test": { + "_info": { + "comment": "Not a test: no fixture fields at all. Beside a fixture which runs, so this is unmistakably a fixture file with one broken entry in it." + } + }, + "c_load_error": { + "_info": { + "fixture-format": "state_test", + "comment": "A state test with no pre state, so loading it throws." + } + }, + "d_bad_root": { + "_info": { + "comment": "Runs and fails on the state root, reached only because the throw above did not abandon the file." + }, + "env": { + "currentBaseFee": "0x0a", + "currentCoinbase": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "currentDifficulty": "0x020000", + "currentGasLimit": "0xff112233445566", + "currentNumber": "0x01", + "currentRandom": "0x0000000000000000000000000000000000000000000000000000000000020000", + "currentTimestamp": "0x03e8" + }, + "post": { + "London": [ + { + "hash": "0x1111111111111111111111111111111111111111111111111111111111111111", + "indexes": { + "data": 0, + "gas": 0, + "value": 0 + }, + "logs": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347" + } + ] + }, + "pre": { + "0x095e7baea6a6c7c4c2dfeb977efac326af552d87": { + "balance": "0x0de0b6b3a7640000", + "code": "0x600160010160005500", + "nonce": "0x00", + "storage": {} + }, + "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba": { + "balance": "0x00", + "code": "0x", + "nonce": "0x01", + "storage": {} + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0x0de0b6b3a7640000", + "code": "0x", + "nonce": "0x00", + "storage": {} + } + }, + "transaction": { + "data": [ + "0x" + ], + "gasLimit": [ + "0x061a80" + ], + "gasPrice": "0x0a", + "nonce": "0x00", + "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "to": "0x095e7baea6a6c7c4c2dfeb977efac326af552d87", + "value": [ + "0x0186a0" + ] + } + } +} diff --git a/test/integration/evmone-cli/test/testcmd_fault/unrecognised_case.json b/test/integration/evmone-cli/test/testcmd_fault/unrecognised_case.json deleted file mode 100644 index e389b69953..0000000000 --- a/test/integration/evmone-cli/test/testcmd_fault/unrecognised_case.json +++ /dev/null @@ -1,67 +0,0 @@ -{ - "a_runs": { - "_info": {}, - "env": { - "currentBaseFee": "0x0a", - "currentCoinbase": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", - "currentDifficulty": "0x020000", - "currentGasLimit": "0xff112233445566", - "currentNumber": "0x01", - "currentRandom": "0x0000000000000000000000000000000000000000000000000000000000020000", - "currentTimestamp": "0x03e8" - }, - "post": { - "London": [ - { - "hash": "0xe8010ce590f401c9d61fef8ab05bea9bcec24281b795e5868809bc4e515aa530", - "indexes": { - "data": 0, - "gas": 0, - "value": 0 - }, - "logs": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347" - } - ] - }, - "pre": { - "0x095e7baea6a6c7c4c2dfeb977efac326af552d87": { - "balance": "0x0de0b6b3a7640000", - "code": "0x600160010160005500", - "nonce": "0x00", - "storage": {} - }, - "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba": { - "balance": "0x00", - "code": "0x", - "nonce": "0x01", - "storage": {} - }, - "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { - "balance": "0x0de0b6b3a7640000", - "code": "0x", - "nonce": "0x00", - "storage": {} - } - }, - "transaction": { - "data": [ - "0x" - ], - "gasLimit": [ - "0x061a80" - ], - "gasPrice": "0x0a", - "nonce": "0x00", - "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", - "to": "0x095e7baea6a6c7c4c2dfeb977efac326af552d87", - "value": [ - "0x0186a0" - ] - } - }, - "b_not_a_fixture": { - "_info": { - "comment": "Not a test: no fixture fields at all. Beside a case which runs, so this is unmistakably a fixture file with one broken case in it." - } - } -} diff --git a/test/integration/evmone-cli/test/testcmd_skipped/all_unsupported.json b/test/integration/evmone-cli/test/testcmd_skipped/all_unsupported.json new file mode 100644 index 0000000000..dba3d299b8 --- /dev/null +++ b/test/integration/evmone-cli/test/testcmd_skipped/all_unsupported.json @@ -0,0 +1,15 @@ +{ + "a_engine": { + "_info": { + "fixture-format": "blockchain_test_engine", + "comment": "Both fixtures are named on their own, so neither reason is lost." + }, + "blocks": [] + }, + "b_engine_x": { + "_info": { + "fixture-format": "blockchain_test_engine_x" + }, + "blocks": [] + } +} diff --git a/test/integration/evmone-cli/test/testcmd_skipped/empty.json b/test/integration/evmone-cli/test/testcmd_skipped/empty.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/test/integration/evmone-cli/test/testcmd_skipped/empty.json @@ -0,0 +1 @@ +{} diff --git a/test/integration/evmone-cli/test/testcmd_skipped/not_an_object.json b/test/integration/evmone-cli/test/testcmd_skipped/not_an_object.json new file mode 100644 index 0000000000..723839b091 --- /dev/null +++ b/test/integration/evmone-cli/test/testcmd_skipped/not_an_object.json @@ -0,0 +1,7 @@ +[ + { + "comment": "A top-level array, so it holds no named fixture. Its one element would be taken for a blockchain test if the array were walked by index.", + "pre": {}, + "blocks": [] + } +] diff --git a/test/integration/evmone-cli/test/testcmd_skipped/partial_shapes.json b/test/integration/evmone-cli/test/testcmd_skipped/partial_shapes.json new file mode 100644 index 0000000000..5d85ea8afa --- /dev/null +++ b/test/integration/evmone-cli/test/testcmd_skipped/partial_shapes.json @@ -0,0 +1,24 @@ +{ + "a_blocks_no_pre": { + "comment": "Carries one of the two keys a blockchain test is named by, so the shape does not name it and nothing else does either.", + "blocks": [] + }, + "b_tx_no_pre": { + "comment": "The same for a state test: a transaction to apply, but no state to apply it to.", + "transaction": { + "data": [ + "0x" + ], + "gasLimit": [ + "0x061a80" + ], + "gasPrice": "0x0a", + "nonce": "0x00", + "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "to": "0x095e7baea6a6c7c4c2dfeb977efac326af552d87", + "value": [ + "0x0186a0" + ] + } + } +} diff --git a/test/integration/evmone-cli/test/testcmd_skipped/pre_alloc.json b/test/integration/evmone-cli/test/testcmd_skipped/pre_alloc.json new file mode 100644 index 0000000000..015cd24e1d --- /dev/null +++ b/test/integration/evmone-cli/test/testcmd_skipped/pre_alloc.json @@ -0,0 +1,27 @@ +{ + "testIds": [ + "tests/ported_static/vmIOandFlowOperations/test_mload.py::test_mload[fork_Osaka-blockchain_test_engine_x]" + ], + "environment": { + "currentCoinbase": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "currentGasLimit": "0xff112233445566", + "currentNumber": "0x01" + }, + "network": "Osaka", + "chainId": 1, + "pre": { + "0x095e7baea6a6c7c4c2dfeb977efac326af552d87": { + "balance": "0x0de0b6b3a7640000", + "code": "0x600160010160005500", + "nonce": "0x00", + "storage": {} + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0x0de0b6b3a7640000", + "code": "0x", + "nonce": "0x00", + "storage": {} + } + }, + "comment": "EEST keeps the pre-allocation its fixtures share beside them, and none of it is a test. Several entries, none of which names a format or carries a shape." +} diff --git a/test/integration/export/CMakeLists.txt b/test/integration/export/CMakeLists.txt index 6659ef9ffd..3f3d14231a 100644 --- a/test/integration/export/CMakeLists.txt +++ b/test/integration/export/CMakeLists.txt @@ -31,7 +31,7 @@ set_tests_properties( add_test( NAME ${PREFIX}/execute_exported_state_tests # TODO: Broken exported tests are filtered out. - COMMAND evmone-statetest ${EXPORT_DIR}/state_tests --ignore state_transition/block + COMMAND evmone-cli test ${EXPORT_DIR}/state_tests --ignore state_transition/block ) set_tests_properties( ${PREFIX}/execute_exported_state_tests PROPERTIES diff --git a/test/statetest/.clang-tidy b/test/statetest/.clang-tidy deleted file mode 100644 index efc628c8e9..0000000000 --- a/test/statetest/.clang-tidy +++ /dev/null @@ -1,3 +0,0 @@ -InheritParentConfig: true -Checks: > - -clang-analyzer-cplusplus.NewDeleteLeaks diff --git a/test/statetest/CMakeLists.txt b/test/statetest/CMakeLists.txt deleted file mode 100644 index 15e07d1618..0000000000 --- a/test/statetest/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -# evmone: Fast Ethereum Virtual Machine implementation -# Copyright 2022 The evmone Authors. -# SPDX-License-Identifier: Apache-2.0 - -add_executable(evmone-statetest) -target_link_libraries(evmone-statetest PRIVATE evmone::testutils evmone evmone-buildinfo CLI11::CLI11) -target_sources( - evmone-statetest PRIVATE - statetest.cpp -) diff --git a/test/statetest/statetest.cpp b/test/statetest/statetest.cpp deleted file mode 100644 index ea14b4ba99..0000000000 --- a/test/statetest/statetest.cpp +++ /dev/null @@ -1,114 +0,0 @@ -// evmone: Fast Ethereum Virtual Machine implementation -// Copyright 2022 The evmone Authors. -// SPDX-License-Identifier: Apache-2.0 - -#include -#include -#include -#include -#include -#include -#include - -namespace fs = std::filesystem; -using evmone::test::TestCase; - -namespace -{ -/// Adds to @p cases one test per fixture file under @p root, which is that file itself when it -/// is not a directory. -void collect_tests(std::vector& cases, const fs::path& root, - const std::optional& filter, std::span ignored, evmc::VM& vm, - bool trace) -{ - // Which cases -k keeps. It selects within the file's test, because naming the cases up front - // would mean loading the whole tree. - const auto selected = [&filter](const evmone::test::StateTransitionTest& test) { - return !filter.has_value() || test.name.find(*filter) != std::string::npos; - }; - - // A file named directly is its own collection; the ignored paths are relative to the - // directory holding it, as they are to a directory named directly. - const auto is_dir = is_directory(root); - auto files = is_dir ? evmone::test::collect_test_files(root) : std::vector{root}; - evmone::test::ignore_test_files(files, is_dir ? root : root.parent_path(), ignored); - - cases.reserve(cases.size() + files.size()); - for (const auto& path : files) - { - // Loaded when the test runs: loading a whole tree up front costs far more. - cases.push_back( - {path.string(), [path, selected, &vm, trace](evmone::test::TestReport& report) { - std::ifstream f{path}; - for (const auto& test : evmone::test::load_state_tests(f)) - { - if (selected(test)) - evmone::test::run_state_test(test, vm, trace, report); - } - }}); - } -} -} // namespace - - -int main(int argc, char* argv[]) -{ - try - { - CLI::App app{"evmone state test runner"}; - - app.set_version_flag("--version", "evmone-statetest " EVMONE_VERSION); - - std::vector paths; - app.add_option("path", paths, - "Path to a test file or a directory of them. Under a directory every .json " - "file except index.json is one test; a file named directly is one test.") - ->required() - ->check(CLI::ExistingPath); - - std::optional filter; - app.add_option("-k", filter, - "Test case name filter. Run only the cases whose name contains the given string."); - - std::vector ignored; - app.add_option("--ignore", ignored, - "Path, relative to a given path, not to collect tests from. May be given more " - "than once. Whole path components are matched, so --ignore bc4895 keeps " - "bc4895-withdrawals.") - // Without this the option is variadic and swallows the positional paths after it. - ->allow_extra_args(false); - - bool collect_only = false; - app.add_flag("--collect-only", collect_only, - "List the path of each collected test, one per line, and exit."); - - bool trace = false; - bool trace_summary = false; - const auto trace_opt = app.add_flag("--trace", trace, "Enable EVM tracing"); - app.add_flag("--trace-summary", trace_summary, "Output trace summary only") - ->excludes(trace_opt); - - CLI11_PARSE(app, argc, argv); - - evmc::VM vm{evmc_create_evmone(), {{"O", "0"}}}; - - if (trace) - { - std::ios::sync_with_stdio(false); - vm.set_option("trace", "1"); - } - - std::vector cases; - for (const auto& p : paths) - collect_tests(cases, p, filter, ignored, vm, trace || trace_summary); - - const evmone::test::RunOptions options{ - .collect_only = collect_only, .progress = !(trace || trace_summary)}; - return evmone::test::run_tests(cases, std::cout, options); - } - catch (const std::exception& ex) - { - std::cerr << ex.what() << "\n"; - return -1; - } -} diff --git a/test/unittests/test_driver_test.cpp b/test/unittests/test_driver_test.cpp index d2208aebd9..8c013927e9 100644 --- a/test/unittests/test_driver_test.cpp +++ b/test/unittests/test_driver_test.cpp @@ -22,6 +22,21 @@ Run run(std::span cases, const RunOptions& options = {}) const auto exit_code = run_tests(cases, out, options); return {exit_code, std::move(out).str()}; } + +/// A test which runs one fixture, as a file holding one does. +TestCase one(const std::string& name, std::function run) +{ + return {name, [name, run = std::move(run)] { return std::vector{run_one(name, run)}; }}; +} + +/// A test whose fixtures are handed their outcomes, running nothing. +TestCase holding(std::string name, std::initializer_list outcomes) +{ + std::vector results; + for (const auto outcome : outcomes) + results.push_back({name + "::case", outcome, "the reason", {}}); + return {std::move(name), [results = std::move(results)] { return results; }}; +} } // namespace TEST(test_driver, nothing_collected) @@ -29,13 +44,13 @@ TEST(test_driver, nothing_collected) const auto [exit_code, output] = run({}); EXPECT_EQ(NOTHING_VERIFIED, 5); // pytest's value, not just whatever we declared. EXPECT_EQ(exit_code, NOTHING_VERIFIED); - EXPECT_NE(output.find("collected 0 tests"), std::string::npos); + EXPECT_NE(output.find("collected 0 files"), std::string::npos); } TEST(test_driver, collect_only_lists_without_running) { bool ran = false; - const std::vector cases{{"a name", [&ran](TestReport&) { ran = true; }}}; + const std::vector cases{one("a name", [&ran](TestReport&) { ran = true; })}; const auto [exit_code, output] = run(cases, {.collect_only = true}); EXPECT_FALSE(ran); @@ -54,10 +69,10 @@ TEST(test_driver, exception_fails_only_its_own_test) { bool last_ran = false; const std::vector cases{ - {"ok", [](TestReport&) {}}, - {"throws", [](TestReport&) { throw std::runtime_error{"the reason"}; }}, - {"unknown", [](TestReport&) { throw 42; }}, // NOLINT(hicpp-exception-baseclass) - {"last", [&last_ran](TestReport&) { last_ran = true; }}, + one("ok", [](TestReport&) {}), + one("throws", [](TestReport&) { throw std::runtime_error{"the reason"}; }), + one("unknown", [](TestReport&) { throw 42; }), // NOLINT(hicpp-exception-baseclass) + one("last", [&last_ran](TestReport&) { last_ran = true; }), }; const auto [exit_code, output] = run(cases); @@ -71,8 +86,8 @@ TEST(test_driver, exception_fails_only_its_own_test) TEST(test_driver, unsupported_feature_skips) { const std::vector cases{ - {"ok", [](TestReport&) {}}, - {"skipped", [](TestReport&) { throw UnsupportedTestFeature{"no support for it"}; }}, + one("ok", [](TestReport&) {}), + one("skipped", [](TestReport&) { throw UnsupportedTestFeature{"no support for it"}; }), }; const auto [exit_code, output] = run(cases); @@ -84,7 +99,7 @@ TEST(test_driver, unsupported_feature_skips) TEST(test_driver, everything_skipped_verifies_nothing) { const std::vector cases{ - {"skipped", [](TestReport&) { throw UnsupportedTestFeature{"no support for it"}; }}}; + one("skipped", [](TestReport&) { throw UnsupportedTestFeature{"no support for it"}; })}; const auto [exit_code, output] = run(cases); EXPECT_EQ(exit_code, NOTHING_VERIFIED); @@ -94,7 +109,7 @@ TEST(test_driver, everything_skipped_verifies_nothing) TEST(test_driver, summary_names_the_check_which_failed) { const std::vector cases{ - {"mismatch", [](TestReport& report) { report.check_eq("a value", 1, 2); }}}; + one("mismatch", [](TestReport& report) { report.check_eq("a value", 1, 2); })}; const auto [exit_code, output] = run(cases); EXPECT_EQ(exit_code, 1); @@ -103,10 +118,10 @@ TEST(test_driver, summary_names_the_check_which_failed) TEST(test_driver, failure_outranks_a_later_exception) { - const std::vector cases{{"both", [](TestReport& report) { - report.check_eq("a value", 1, 2); - throw std::runtime_error{"gave up afterwards"}; - }}}; + const std::vector cases{one("both", [](TestReport& report) { + report.check_eq("a value", 1, 2); + throw std::runtime_error{"gave up afterwards"}; + })}; const auto [exit_code, output] = run(cases); EXPECT_EQ(exit_code, TESTS_FAILED); @@ -116,10 +131,10 @@ TEST(test_driver, failure_outranks_a_later_exception) TEST(test_driver, failure_outranks_a_later_skip) { - const std::vector cases{{"both", [](TestReport& report) { - report.check_eq("a value", 1, 2); - throw UnsupportedTestFeature{"gave up afterwards"}; - }}}; + const std::vector cases{one("both", [](TestReport& report) { + report.check_eq("a value", 1, 2); + throw UnsupportedTestFeature{"gave up afterwards"}; + })}; const auto [exit_code, output] = run(cases); EXPECT_EQ(exit_code, 1); @@ -127,3 +142,54 @@ TEST(test_driver, failure_outranks_a_later_skip) // The summary names the check which failed, not what the test then gave up on. EXPECT_NE(output.find("FAILED both - a value"), std::string::npos); } + +TEST(test_driver, a_file_counts_once_however_many_fixtures_it_holds) +{ + const std::vector cases{ + holding("a file", {Outcome::passed, Outcome::passed, Outcome::passed})}; + + const auto [exit_code, output] = run(cases); + EXPECT_EQ(exit_code, SUCCESS); + EXPECT_NE(output.find("collected 1 file"), std::string::npos); + EXPECT_NE(output.find("1 passed"), std::string::npos); +} + +TEST(test_driver, a_declined_fixture_is_named_though_its_file_passed) +{ + const std::vector cases{holding("a file", {Outcome::passed, Outcome::skipped})}; + + const auto [exit_code, output] = run(cases); + EXPECT_EQ(exit_code, SUCCESS); + // The file's own verdict says nothing about what it declined, so the fixture is named. + EXPECT_NE(output.find("1 passed in"), std::string::npos); + EXPECT_NE(output.find("SKIPPED a file::case - the reason"), std::string::npos); +} + +TEST(test_driver, one_failed_fixture_fails_its_file) +{ + const std::vector cases{ + holding("a file", {Outcome::passed, Outcome::failed, Outcome::skipped})}; + + const auto [exit_code, output] = run(cases); + EXPECT_EQ(exit_code, TESTS_FAILED); + EXPECT_NE(output.find("1 failed, 0 passed"), std::string::npos); +} + +TEST(test_driver, a_file_is_skipped_only_when_nothing_in_it_ran) +{ + const std::vector cases{holding("a file", {Outcome::skipped, Outcome::skipped})}; + + const auto [exit_code, output] = run(cases); + EXPECT_EQ(exit_code, NOTHING_VERIFIED); + EXPECT_NE(output.find("0 passed, 1 skipped"), std::string::npos); +} + +TEST(test_driver, a_file_whose_fixtures_were_all_filtered_out_verifies_nothing) +{ + // A filter selecting nothing leaves no result at all, which is not a pass. + const std::vector cases{holding("a file", {})}; + + const auto [exit_code, output] = run(cases); + EXPECT_EQ(exit_code, NOTHING_VERIFIED); + EXPECT_NE(output.find("0 passed, 1 skipped"), std::string::npos); +} diff --git a/test/utils/test_collector.cpp b/test/utils/test_collector.cpp index c327b2f30b..eded5612e9 100644 --- a/test/utils/test_collector.cpp +++ b/test/utils/test_collector.cpp @@ -50,4 +50,31 @@ void ignore_test_files( ignored, [&relative](const fs::path& prefix) { return is_under(relative, prefix); }); }); } + +void collect_tests( + std::vector& cases, const fs::path& root, const RunOptions& options, evmc::VM& vm) +{ + // A file named directly is its own collection; the ignored paths are relative to the + // directory holding it, as they are to a directory named directly. + const auto is_dir = is_directory(root); + auto files = is_dir ? collect_test_files(root) : std::vector{root}; + ignore_test_files(files, is_dir ? root : root.parent_path(), options.ignored); + + cases.reserve(cases.size() + files.size()); + for (const auto& path : files) + { + // Loaded when the test runs: loading a whole tree up front costs far more. + cases.push_back( + {path.string(), [path, &options, &vm] { return run_fixture_file(path, options, vm); }}); + } +} + +int test( + evmc::VM& vm, std::span roots, const RunOptions& options, std::ostream& out) +{ + std::vector cases; + for (const auto& root : roots) + collect_tests(cases, root, options, vm); + return run_tests(cases, out, options); +} } // namespace evmone::test diff --git a/test/utils/test_collector.hpp b/test/utils/test_collector.hpp index cd5269199d..6ecd1e4bfb 100644 --- a/test/utils/test_collector.hpp +++ b/test/utils/test_collector.hpp @@ -3,6 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 #pragma once +#include #include #include #include @@ -18,4 +19,15 @@ namespace evmone::test /// @p root. Whole path components are matched, so ignoring "bc4895" keeps "bc4895-withdrawals". void ignore_test_files(std::vector& files, const std::filesystem::path& root, std::span ignored); + +/// Adds to @p cases one test per fixture file under @p root, which is that file itself when it +/// is not a directory. The tests hold @p options and @p vm by reference, so both must outlive +/// them. +void collect_tests(std::vector& cases, const std::filesystem::path& root, + const RunOptions& options, evmc::VM& vm); + +/// Runs the Ethereum tests collected under every path in @p roots, reporting to @p out. +/// Returns the process exit code. +[[nodiscard]] int test(evmc::VM& vm, std::span roots, + const RunOptions& options, std::ostream& out); } // namespace evmone::test diff --git a/test/utils/test_driver.cpp b/test/utils/test_driver.cpp index 5ad47d8382..713986448c 100644 --- a/test/utils/test_driver.cpp +++ b/test/utils/test_driver.cpp @@ -3,27 +3,25 @@ // SPDX-License-Identifier: Apache-2.0 #include "test_driver.hpp" +#include +#include +#include #include +#include #include #include #include namespace evmone::test { +namespace fs = std::filesystem; + namespace { /// The report is laid out like pytest's. constexpr int LINE_WIDTH = 72; constexpr int PROGRESS_WIDTH = 60; -/// The outcome of one test, spelled as the progress character for it. -enum class Outcome : char -{ - passed = '.', - failed = 'F', - skipped = 's', -}; - void banner(std::ostream& out, std::string_view title, char fill = '=') { const auto padding = LINE_WIDTH - static_cast(title.size()) - 2; @@ -32,13 +30,12 @@ void banner(std::ostream& out, std::string_view title, char fill = '=') << std::string(static_cast(std::max(padding - left, 1)), fill) << '\n'; } -/// A test which did not pass: what the summary says about it and what it recorded. +/// A file with something to report: how it counts, and every fixture of it which did not pass. struct Note { - Outcome outcome; std::string name; - std::string reason; - std::vector failures; + Outcome outcome; + std::vector results; }; /// One progress character per test, wrapped, each line ending in the percentage done. @@ -65,8 +62,117 @@ class Progress m_column = 0; } }; + +/// What this tool makes of one fixture. +enum class Format +{ + state_test, + blockchain_test, + unsupported, ///< A fixture, in a format this tool does not run. + not_a_test, ///< Not a fixture at all. +}; + +Format classify(const json::json& fixture) +{ + if (const auto info = fixture.find("_info"); info != fixture.end()) + { + if (const auto format = info->find("fixture-format"); format != info->end()) + { + if (*format == "state_test") + return Format::state_test; + if (*format == "blockchain_test") + return Format::blockchain_test; + return Format::unsupported; + } + } + // Nothing declares the format: a hand-written or pre-EEST file, or an "_info" without one. + // Each shape is named by the state it starts from and what is applied to it, never by what + // it expects, so a fixture whose expectations are missing is still a test and is run. + // Anything else is not a test at all, as EEST's shared pre-allocation is not. + if (fixture.contains("pre") && fixture.contains("blocks")) + return Format::blockchain_test; + if (fixture.contains("pre") && fixture.contains("transaction")) + return Format::state_test; + return Format::not_a_test; +} + +/// Parses the fixture file at @p path. Throws UnsupportedTestFeature for a file with no +/// fixture in it: EEST keeps its shared pre-allocation and an index beside the fixtures, and +/// neither is a test. +json::json load_fixture_file(const fs::path& path) +{ + std::ifstream f{path}; + const auto contents = json::json::parse(f); + // A document which is not an object holds none either: items() would walk an array by + // index, naming its elements "0", "1", ... + if (!contents.is_object() || std::ranges::none_of(contents.items(), [](const auto& i) { + return classify(i.value()) != Format::not_a_test; + })) + throw UnsupportedTestFeature{"not a test"}; + return contents; +} + +/// Runs one fixture of a fixture file. +void run_fixture(const std::string& name, const json::json& fixture, const RunOptions& options, + evmc::VM& vm, TestReport& report) +{ + // The runners rename theirs; this names a failure to load, or a fixture which is not one. + report.start_case(name); + switch (classify(fixture)) + { + case Format::state_test: + run_state_test(make_state_test(name, fixture), vm, options.trace_summary, report); + break; + case Format::blockchain_test: + run_blockchain_test(make_blockchain_test(name, fixture), vm, report); + break; + case Format::unsupported: + throw UnsupportedTestFeature{ + "unsupported fixture format: " + fixture.at("_info").at("fixture-format").dump()}; + case Format::not_a_test: + // The rest of the file holds fixtures, so this one is broken. + report.fail("not a test"); + break; + } +} + } // namespace +Result run_one(std::string name, const std::function& run) +{ + Result result{.name = std::move(name)}; + TestReport report{[&result](const Failure& failure) { result.failures.push_back(failure); }}; + report.start_case(result.name); + + try + { + run(report); + } + catch (const UnsupportedTestFeature& ex) + { + result.outcome = Outcome::skipped; + result.reason = ex.what(); + } + catch (const std::exception& ex) + { + // One unloadable fixture in a tree of thousands fails its own test, not the run. + report.fail(concat("exception: ", ex.what())); + } + catch (...) + { + report.fail("exception not derived from std::exception"); + } + + // The failures are in the order they happened, so the first is what the test is reported + // as: one recorded before the run gave up outranks it. + if (!result.failures.empty()) + { + result.outcome = Outcome::failed; + result.reason = result.failures.front().what; + } + return result; +} + int run_tests(std::span cases, std::ostream& out, const RunOptions& options) { if (options.collect_only) @@ -79,68 +185,57 @@ int run_tests(std::span cases, std::ostream& out, const RunOptio const auto started = std::chrono::steady_clock::now(); banner(out, "test session starts"); - out << "collected " << cases.size() << (cases.size() == 1 ? " test\n\n" : " tests\n\n"); + out << "collected " << cases.size() << (cases.size() == 1 ? " file\n\n" : " files\n\n"); + // Held until the run ends, as pytest holds them, so nothing interleaves. std::vector notes; Progress row{out, cases.size()}; + size_t failed = 0; + size_t skipped = 0; + size_t passed = 0; for (const auto& test : cases) { - // Held until the run ends, as pytest holds them, so nothing interleaves. - std::vector failures; - TestReport report{[&failures](const Failure& failure) { failures.push_back(failure); }}; - report.start_case(test.name); - - auto outcome = Outcome::passed; - std::string reason; - std::string exception_reason; - if (!options.progress) - out << test.name << '\n'; // The only thing naming what the test prints next. out << std::flush; - try - { - test.run(report); - } - catch (const UnsupportedTestFeature& ex) - { - outcome = Outcome::skipped; - reason = ex.what(); - } - catch (const std::exception& ex) - { - // One unloadable fixture in a tree of thousands fails its own test, not the run. - report.fail("exception", ex.what()); - exception_reason = concat("exception: ", ex.what()); - } - catch (...) - { - report.fail("exception", "not derived from std::exception"); - exception_reason = "exception not derived from std::exception"; - } + + auto results = test.run(); // A test writes its own output, an EVM trace above all, to another stream. std::clog << std::flush; - // A recorded failure outranks giving up afterwards, in the summary too: the exception - // is the reason only when nothing failed before it threw. - if (!failures.empty()) - { + // The file counts once, for the worst its fixtures reached. It is skipped only when + // nothing in it ran at all, so one fixture running is enough to give it a verdict. + static constexpr auto is = [](Outcome outcome) { + return [outcome](const Result& result) { return result.outcome == outcome; }; + }; + auto outcome = Outcome::passed; + if (std::ranges::any_of(results, is(Outcome::failed))) outcome = Outcome::failed; - reason = failures.size() == 1 && !exception_reason.empty() ? - std::move(exception_reason) : - failures.front().what; + else if (std::ranges::none_of(results, is(Outcome::passed))) + outcome = Outcome::skipped; + + switch (outcome) + { + case Outcome::passed: + ++passed; + break; + case Outcome::failed: + ++failed; + break; + case Outcome::skipped: + ++skipped; + break; } - if (outcome != Outcome::passed) - notes.push_back({outcome, test.name, std::move(reason), std::move(failures)}); + + // Every fixture which did not pass is named, including one declined by a file which + // passed on the fixtures beside it. Otherwise it would vanish from a green run. + std::erase_if(results, is(Outcome::passed)); + if (!results.empty()) + notes.push_back({test.name, outcome, std::move(results)}); if (options.progress) row.advance(outcome); } - // Every test which did not pass left exactly one note, so the counts follow from them. - const auto failed = std::ranges::count(notes, Outcome::failed, &Note::outcome); - const auto skipped = std::ranges::count(notes, Outcome::skipped, &Note::outcome); - const auto passed = cases.size() - notes.size(); - if (failed != 0) { out << '\n'; @@ -150,8 +245,11 @@ int run_tests(std::span cases, std::ostream& out, const RunOptio if (note.outcome != Outcome::failed) continue; banner(out, note.name, '_'); - for (const auto& failure : note.failures) - out << failure << '\n'; + for (const auto& result : note.results) + { + for (const auto& failure : result.failures) + out << failure << '\n'; + } } } @@ -161,10 +259,13 @@ int run_tests(std::span cases, std::ostream& out, const RunOptio banner(out, "short test summary info"); for (const auto& note : notes) { - out << (note.outcome == Outcome::failed ? "FAILED " : "SKIPPED ") << note.name; - if (!note.reason.empty()) - out << " - " << note.reason; - out << '\n'; + for (const auto& result : note.results) + { + // Only a passed result has no reason, and those were erased above. + assert(!result.reason.empty()); + out << (result.outcome == Outcome::failed ? "FAILED " : "SKIPPED ") << result.name + << " - " << result.reason << '\n'; + } } } @@ -181,8 +282,26 @@ int run_tests(std::span cases, std::ostream& out, const RunOptio if (failed != 0) return TESTS_FAILED; - // No test passed: nothing was collected, or every test was skipped. A test which holds no - // case of its own still counts as passed, which this does not change. return passed == 0 ? NOTHING_VERIFIED : SUCCESS; } + +std::vector run_fixture_file(const fs::path& path, const RunOptions& options, evmc::VM& vm) +{ + json::json contents; + if (auto loaded = + run_one(path.string(), [&](TestReport&) { contents = load_fixture_file(path); }); + loaded.outcome != Outcome::passed) + return {std::move(loaded)}; + + std::vector results; + for (const auto& [name, fixture] : contents.items()) + { + if (!options.selects(name)) + continue; + results.push_back(run_one(path.string() + "::" + name, + [&](TestReport& report) { run_fixture(name, fixture, options, vm, report); })); + } + return results; +} + } // namespace evmone::test diff --git a/test/utils/test_driver.hpp b/test/utils/test_driver.hpp index 2694b48680..908aff786b 100644 --- a/test/utils/test_driver.hpp +++ b/test/utils/test_driver.hpp @@ -3,7 +3,12 @@ // SPDX-License-Identifier: Apache-2.0 #pragma once +#include #include +#include +#include +#include +#include namespace evmone::test { @@ -17,24 +22,67 @@ constexpr int TESTS_FAILED = 1; /// value for the first of those; a test skipped has verified no more than a missing one. constexpr int NOTHING_VERIFIED = 5; +/// The verdict on a fixture or on a file, spelled as the file's progress character. +enum class Outcome : char +{ + passed = '.', + failed = 'F', + skipped = 's', +}; + +/// What running one fixture produced. +struct Result +{ + /// The fixture, as "::", or the file alone when it never got as far as one. + std::string name; + + Outcome outcome = Outcome::passed; + + /// Why it did not pass. Empty when it did. + std::string reason; + + std::vector failures; +}; + /// A single test: its name and how to run it. struct TestCase { std::string name; - /// Executes the test, recording what did not hold in the report. - std::function run; + /// Executes the test, returning what each of its fixtures produced. A test which never got + /// as far as a fixture returns the one result which says so, rather than throwing. + std::function()> run; }; -/// How to run and what to report. +/// Runs @p run under a report of its own and says what it produced. What the run recorded +/// outranks how it ended: an exception is the reason only when nothing failed before it threw. +[[nodiscard]] Result run_one(std::string name, const std::function& run); + +/// What to collect, what to run of it, and what to report. struct RunOptions { /// List the tests instead of running them. bool collect_only = false; - /// Mark each test with a progress character rather than report its name. A progress line - /// has no terminating newline, so anything a test prints itself would continue it. + /// Mark each test with a progress character. A progress line has no terminating newline, + /// so anything a test prints itself would continue it. bool progress = true; + + /// Run only the fixtures whose name contains this. + std::optional name_filter; + + /// Paths, relative to a test directory, not to collect tests from. + std::vector ignored; + + /// Report each state test's execution summary on the trace stream. Blockchain tests + /// have no summary to report. + bool trace_summary = false; + + /// Whether the name filter, if any, keeps the fixture called @p name. + [[nodiscard]] bool selects(const std::string& name) const noexcept + { + return !name_filter.has_value() || name.find(*name_filter) != std::string::npos; + } }; /// Runs @p cases, reports to @p out and returns the process exit code. @@ -43,4 +91,10 @@ struct RunOptions /// got. [[nodiscard]] int run_tests( std::span cases, std::ostream& out, const RunOptions& options = {}); + +/// Runs every selected fixture of one fixture file, which together are one test, and returns +/// what each produced. A file which holds no fixture, or does not parse, is the one result. +[[nodiscard]] std::vector run_fixture_file( + const std::filesystem::path& path, const RunOptions& options, evmc::VM& vm); + } // namespace evmone::test diff --git a/tools/evmone/main.cpp b/tools/evmone/main.cpp index e5e6c24ece..2e1025a635 100644 --- a/tools/evmone/main.cpp +++ b/tools/evmone/main.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -151,6 +152,45 @@ int exec_t8n_cmd(evmc::VM& vm, const T8nOptions& opts) evmone::tooling::t8n(vm, args); return 0; } + +const CLI::App& setup_test_cmd( + CLI::App& app, std::vector& paths, evmone::test::RunOptions& opts) +{ + auto& cmd = *app.add_subcommand("test", "Run Ethereum tests")->fallthrough(); + cmd.add_option("path", paths, + "Test file or directory. Every fixture file is one test: under a directory, each " + ".json file except index.json.") + ->required() + ->check(CLI::ExistingPath); + cmd.add_option("-k", opts.name_filter, "Run only the fixtures whose name contains this."); + cmd.add_option("--ignore", opts.ignored, + "Path not to collect tests from, relative to each directory given, or to the " + "directory holding a file given. May be given more than once. Whole path components " + "are matched, so --ignore bc4895 keeps bc4895-withdrawals.") + // Without this the option is variadic and swallows the positional paths after it. + ->allow_extra_args(false); + cmd.add_flag( + "--collect-only", opts.collect_only, "List each collected test, one per line, and exit."); + cmd.add_flag("--trace-summary", opts.trace_summary, + "Report each state test's execution summary, as --trace also does. Blockchain tests " + "have no summary to report."); + return cmd; +} + +int exec_test_cmd(evmc::VM& vm, std::span paths, evmone::test::RunOptions opts, + bool trace, bool histogram) +{ + // main() has switched the tracer on already. Its line per instruction is worth + // unsynchronising the streams for, and anything it writes per test would run into the + // progress row, as a summary would. + if (trace) + std::ios::sync_with_stdio(false); + opts.trace_summary |= trace; + opts.progress = !(opts.trace_summary || histogram); + + return evmone::test::test(vm, paths, opts, std::cout); +} + } // namespace int main(int argc, const char* const* argv) noexcept @@ -198,6 +238,10 @@ int main(int argc, const char* const* argv) noexcept T8nOptions t8n_opts; const auto& t8n_cmd = setup_t8n_cmd(app, t8n_opts); + std::vector test_paths; + evmone::test::RunOptions test_opts; + const auto& test_cmd = setup_test_cmd(app, test_paths, test_opts); + try { app.parse(argc, argv); @@ -218,6 +262,9 @@ int main(int argc, const char* const* argv) noexcept if (t8n_cmd) return exec_t8n_cmd(vm, t8n_opts); + if (test_cmd) + return exec_test_cmd(vm, test_paths, test_opts, trace, histogram); + return 0; } catch (const CLI::ParseError& e)