From 6c9f3ecca9e25001a74837e29948a481a5d2d905 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=81=E5=B2=91?= <167747754+vzer200@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:41:05 +0800 Subject: [PATCH 1/3] Keep CMake reader/writer test artifacts in the build tree --- CONTRIBUTING.md | 6 +++++- src/jsontestrunner/CMakeLists.txt | 13 +++++++------ test/runjsontests.py | 22 ++++++++++++++++------ 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5f5c032a8..6e6f467d2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -94,7 +94,11 @@ See the examples `test_complex_01.json` and `test_complex_01.expected` to better ## Understanding reader/writer test output -When a test is run, output files are generated beside the input test files. Below is a short description of the content of each file: +When running tests manually, output files are generated beside the input test files by default. +Pass `--output-dir ` to `runjsontests.py` to keep copied inputs and generated output +in another directory. Expected results are still read from the original input directory. +CMake uses this option to keep test artifacts in the build tree, so the source tree can be read-only. +Below is a short description of the content of each file: * `test_complex_01.json`: input JSON document. * `test_complex_01.expected`: flattened JSON element tree used to check if diff --git a/src/jsontestrunner/CMakeLists.txt b/src/jsontestrunner/CMakeLists.txt index 7f536d318..0802018ef 100644 --- a/src/jsontestrunner/CMakeLists.txt +++ b/src/jsontestrunner/CMakeLists.txt @@ -30,26 +30,27 @@ if(PYTHONINTERP_FOUND) # Run end to end parser/writer tests set(TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../test) set(RUNJSONTESTS_PATH ${TEST_DIR}/runjsontests.py) + set(TEST_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/test-output") # Run unit tests in post-build # (default cmake workflow hides away the test result into a file, resulting in poor dev workflow?!?) add_custom_target(jsoncpp_readerwriter_tests - "${PYTHON_EXECUTABLE}" -B "${RUNJSONTESTS_PATH}" $ "${TEST_DIR}/data" + "${PYTHON_EXECUTABLE}" -B "${RUNJSONTESTS_PATH}" --output-dir "${TEST_OUTPUT_DIR}" $ "${TEST_DIR}/data" DEPENDS jsontestrunner_exe jsoncpp_test ) add_custom_target(jsoncpp_check DEPENDS jsoncpp_readerwriter_tests) ## Create tests for dashboard submission, allows easy review of CI results https://my.cdash.org/index.php?project=jsoncpp add_test(NAME jsoncpp_readerwriter - COMMAND "${PYTHON_EXECUTABLE}" -B "${RUNJSONTESTS_PATH}" $ "${TEST_DIR}/data" - WORKING_DIRECTORY "${TEST_DIR}/data" + COMMAND "${PYTHON_EXECUTABLE}" -B "${RUNJSONTESTS_PATH}" --output-dir "${TEST_OUTPUT_DIR}" $ "${TEST_DIR}/data" + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" ) add_test(NAME jsoncpp_readerwriter_json_checker - COMMAND "${PYTHON_EXECUTABLE}" -B "${RUNJSONTESTS_PATH}" --with-json-checker $ "${TEST_DIR}/data" - WORKING_DIRECTORY "${TEST_DIR}/data" + COMMAND "${PYTHON_EXECUTABLE}" -B "${RUNJSONTESTS_PATH}" --with-json-checker --output-dir "${TEST_OUTPUT_DIR}" $ "${TEST_DIR}/data" + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" ) - # Both tests write .actual/.actual-rewrite along with test data, need to prevent collision when running tests via ctest -j + # Both tests share output files, need to prevent collision when running tests via ctest -j set_tests_properties(jsoncpp_readerwriter jsoncpp_readerwriter_json_checker PROPERTIES RESOURCE_LOCK "test_data_files" ) diff --git a/test/runjsontests.py b/test/runjsontests.py index 14275ec22..ae12ed54f 100644 --- a/test/runjsontests.py +++ b/test/runjsontests.py @@ -11,6 +11,7 @@ import os import os.path import optparse +import shutil VALGRIND_CMD = 'valgrind --tool=memcheck --leak-check=yes --undef-value-errors=yes ' @@ -68,9 +69,11 @@ def __init__(self, msg): def runAllTests(jsontest_executable_path, input_path = None, use_valgrind=False, with_json_checker=False, - writerClass='StyledWriter'): + writerClass='StyledWriter', output_path=None): if not input_path: input_path = os.path.join(os.getcwd(), 'data') + if output_path and not os.path.isdir(output_path): + os.makedirs(output_path) if os.path.isdir(input_path): tests = [ @@ -113,12 +116,17 @@ def runAllTests(jsontest_executable_path, input_path = None, is_json_checker_test = os.path.basename(os.path.dirname(input_path)) == "jsonchecker" is_parse_only = is_json_checker_test or expect_failure is_strict_test = ('_strict_' in os.path.basename(input_path)) or is_json_checker_test + run_input_path = input_path + if output_path and not is_parse_only: + run_input_path = os.path.join(output_path, os.path.basename(input_path)) + if not os.path.exists(run_input_path) or not os.path.samefile(input_path, run_input_path): + shutil.copyfile(input_path, run_input_path) print('TESTING:', input_path, end=' ') options = is_parse_only and '--parse-only' or '' options += is_strict_test and ' --strict' or '' options += ' --json-writer %s'%writerClass cmd = '%s%s %s "%s"' % ( valgrind_path, jsontest_executable_path, options, - input_path) + run_input_path) status, process_output = getStatusOutput(cmd) if is_parse_only: if expect_failure: @@ -135,7 +143,7 @@ def runAllTests(jsontest_executable_path, input_path = None, else: print('OK') else: - base_path = os.path.splitext(input_path)[0] + base_path = os.path.splitext(run_input_path)[0] actual_output = safeReadFile(base_path + '.actual') actual_rewrite_output = safeReadFile(base_path + '.actual-rewrite') open(base_path + '.process-output', 'wt', encoding = 'utf-8').write(process_output) @@ -175,6 +183,8 @@ def main(): parser.add_option("-c", "--with-json-checker", action="store_true", dest="with_json_checker", default=False, help="run all the tests from the official JSONChecker test suite of json.org") + parser.add_option("--output-dir", dest="output_path", + help="write reader/writer test artifacts here instead of beside the input files") parser.enable_interspersed_args() options, args = parser.parse_args() @@ -190,15 +200,15 @@ def main(): runAllTests(jsontest_executable_path, input_path, use_valgrind=options.valgrind, with_json_checker=options.with_json_checker, - writerClass='StyledWriter') + writerClass='StyledWriter', output_path=options.output_path) runAllTests(jsontest_executable_path, input_path, use_valgrind=options.valgrind, with_json_checker=options.with_json_checker, - writerClass='StyledStreamWriter') + writerClass='StyledStreamWriter', output_path=options.output_path) runAllTests(jsontest_executable_path, input_path, use_valgrind=options.valgrind, with_json_checker=options.with_json_checker, - writerClass='BuiltStyledStreamWriter') + writerClass='BuiltStyledStreamWriter', output_path=options.output_path) if __name__ == '__main__': try: From e11949d7109d05e7088713d663e7398cae3d1f1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=81=E5=B2=91?= <167747754+vzer200@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:51:04 +0800 Subject: [PATCH 2/3] Allow concurrent creation of the test output directory --- test/runjsontests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/runjsontests.py b/test/runjsontests.py index ae12ed54f..3caaec367 100644 --- a/test/runjsontests.py +++ b/test/runjsontests.py @@ -72,8 +72,8 @@ def runAllTests(jsontest_executable_path, input_path = None, writerClass='StyledWriter', output_path=None): if not input_path: input_path = os.path.join(os.getcwd(), 'data') - if output_path and not os.path.isdir(output_path): - os.makedirs(output_path) + if output_path: + os.makedirs(output_path, exist_ok=True) if os.path.isdir(input_path): tests = [ From 61c822e296c9809d2a304078b7b4098ff273ea95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=81=E5=B2=91?= <167747754+vzer200@users.noreply.github.com> Date: Tue, 8 Sep 2026 17:44:34 +0800 Subject: [PATCH 3/3] Separate CTest and custom target output files --- src/jsontestrunner/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/jsontestrunner/CMakeLists.txt b/src/jsontestrunner/CMakeLists.txt index 0802018ef..8e090cf86 100644 --- a/src/jsontestrunner/CMakeLists.txt +++ b/src/jsontestrunner/CMakeLists.txt @@ -35,18 +35,18 @@ if(PYTHONINTERP_FOUND) # Run unit tests in post-build # (default cmake workflow hides away the test result into a file, resulting in poor dev workflow?!?) add_custom_target(jsoncpp_readerwriter_tests - "${PYTHON_EXECUTABLE}" -B "${RUNJSONTESTS_PATH}" --output-dir "${TEST_OUTPUT_DIR}" $ "${TEST_DIR}/data" + "${PYTHON_EXECUTABLE}" -B "${RUNJSONTESTS_PATH}" --output-dir "${TEST_OUTPUT_DIR}/jsoncpp_check" $ "${TEST_DIR}/data" DEPENDS jsontestrunner_exe jsoncpp_test ) add_custom_target(jsoncpp_check DEPENDS jsoncpp_readerwriter_tests) ## Create tests for dashboard submission, allows easy review of CI results https://my.cdash.org/index.php?project=jsoncpp add_test(NAME jsoncpp_readerwriter - COMMAND "${PYTHON_EXECUTABLE}" -B "${RUNJSONTESTS_PATH}" --output-dir "${TEST_OUTPUT_DIR}" $ "${TEST_DIR}/data" + COMMAND "${PYTHON_EXECUTABLE}" -B "${RUNJSONTESTS_PATH}" --output-dir "${TEST_OUTPUT_DIR}/ctest" $ "${TEST_DIR}/data" WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" ) add_test(NAME jsoncpp_readerwriter_json_checker - COMMAND "${PYTHON_EXECUTABLE}" -B "${RUNJSONTESTS_PATH}" --with-json-checker --output-dir "${TEST_OUTPUT_DIR}" $ "${TEST_DIR}/data" + COMMAND "${PYTHON_EXECUTABLE}" -B "${RUNJSONTESTS_PATH}" --with-json-checker --output-dir "${TEST_OUTPUT_DIR}/ctest" $ "${TEST_DIR}/data" WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" )