diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5f5c032a8..6e6f467d2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -94,7 +94,11 @@ See the examples `test_complex_01.json` and `test_complex_01.expected` to better ## Understanding reader/writer test output -When a test is run, output files are generated beside the input test files. Below is a short description of the content of each file: +When running tests manually, output files are generated beside the input test files by default. +Pass `--output-dir ` to `runjsontests.py` to keep copied inputs and generated output +in another directory. Expected results are still read from the original input directory. +CMake uses this option to keep test artifacts in the build tree, so the source tree can be read-only. +Below is a short description of the content of each file: * `test_complex_01.json`: input JSON document. * `test_complex_01.expected`: flattened JSON element tree used to check if diff --git a/src/jsontestrunner/CMakeLists.txt b/src/jsontestrunner/CMakeLists.txt index 7f536d318..8e090cf86 100644 --- a/src/jsontestrunner/CMakeLists.txt +++ b/src/jsontestrunner/CMakeLists.txt @@ -30,26 +30,27 @@ if(PYTHONINTERP_FOUND) # Run end to end parser/writer tests set(TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../test) set(RUNJSONTESTS_PATH ${TEST_DIR}/runjsontests.py) + set(TEST_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/test-output") # Run unit tests in post-build # (default cmake workflow hides away the test result into a file, resulting in poor dev workflow?!?) add_custom_target(jsoncpp_readerwriter_tests - "${PYTHON_EXECUTABLE}" -B "${RUNJSONTESTS_PATH}" $ "${TEST_DIR}/data" + "${PYTHON_EXECUTABLE}" -B "${RUNJSONTESTS_PATH}" --output-dir "${TEST_OUTPUT_DIR}/jsoncpp_check" $ "${TEST_DIR}/data" DEPENDS jsontestrunner_exe jsoncpp_test ) add_custom_target(jsoncpp_check DEPENDS jsoncpp_readerwriter_tests) ## Create tests for dashboard submission, allows easy review of CI results https://my.cdash.org/index.php?project=jsoncpp add_test(NAME jsoncpp_readerwriter - COMMAND "${PYTHON_EXECUTABLE}" -B "${RUNJSONTESTS_PATH}" $ "${TEST_DIR}/data" - WORKING_DIRECTORY "${TEST_DIR}/data" + COMMAND "${PYTHON_EXECUTABLE}" -B "${RUNJSONTESTS_PATH}" --output-dir "${TEST_OUTPUT_DIR}/ctest" $ "${TEST_DIR}/data" + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" ) add_test(NAME jsoncpp_readerwriter_json_checker - COMMAND "${PYTHON_EXECUTABLE}" -B "${RUNJSONTESTS_PATH}" --with-json-checker $ "${TEST_DIR}/data" - WORKING_DIRECTORY "${TEST_DIR}/data" + COMMAND "${PYTHON_EXECUTABLE}" -B "${RUNJSONTESTS_PATH}" --with-json-checker --output-dir "${TEST_OUTPUT_DIR}/ctest" $ "${TEST_DIR}/data" + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" ) - # Both tests write .actual/.actual-rewrite along with test data, need to prevent collision when running tests via ctest -j + # Both tests share output files, need to prevent collision when running tests via ctest -j set_tests_properties(jsoncpp_readerwriter jsoncpp_readerwriter_json_checker PROPERTIES RESOURCE_LOCK "test_data_files" ) diff --git a/test/runjsontests.py b/test/runjsontests.py index 14275ec22..3caaec367 100644 --- a/test/runjsontests.py +++ b/test/runjsontests.py @@ -11,6 +11,7 @@ import os import os.path import optparse +import shutil VALGRIND_CMD = 'valgrind --tool=memcheck --leak-check=yes --undef-value-errors=yes ' @@ -68,9 +69,11 @@ def __init__(self, msg): def runAllTests(jsontest_executable_path, input_path = None, use_valgrind=False, with_json_checker=False, - writerClass='StyledWriter'): + writerClass='StyledWriter', output_path=None): if not input_path: input_path = os.path.join(os.getcwd(), 'data') + if output_path: + os.makedirs(output_path, exist_ok=True) if os.path.isdir(input_path): tests = [ @@ -113,12 +116,17 @@ def runAllTests(jsontest_executable_path, input_path = None, is_json_checker_test = os.path.basename(os.path.dirname(input_path)) == "jsonchecker" is_parse_only = is_json_checker_test or expect_failure is_strict_test = ('_strict_' in os.path.basename(input_path)) or is_json_checker_test + run_input_path = input_path + if output_path and not is_parse_only: + run_input_path = os.path.join(output_path, os.path.basename(input_path)) + if not os.path.exists(run_input_path) or not os.path.samefile(input_path, run_input_path): + shutil.copyfile(input_path, run_input_path) print('TESTING:', input_path, end=' ') options = is_parse_only and '--parse-only' or '' options += is_strict_test and ' --strict' or '' options += ' --json-writer %s'%writerClass cmd = '%s%s %s "%s"' % ( valgrind_path, jsontest_executable_path, options, - input_path) + run_input_path) status, process_output = getStatusOutput(cmd) if is_parse_only: if expect_failure: @@ -135,7 +143,7 @@ def runAllTests(jsontest_executable_path, input_path = None, else: print('OK') else: - base_path = os.path.splitext(input_path)[0] + base_path = os.path.splitext(run_input_path)[0] actual_output = safeReadFile(base_path + '.actual') actual_rewrite_output = safeReadFile(base_path + '.actual-rewrite') open(base_path + '.process-output', 'wt', encoding = 'utf-8').write(process_output) @@ -175,6 +183,8 @@ def main(): parser.add_option("-c", "--with-json-checker", action="store_true", dest="with_json_checker", default=False, help="run all the tests from the official JSONChecker test suite of json.org") + parser.add_option("--output-dir", dest="output_path", + help="write reader/writer test artifacts here instead of beside the input files") parser.enable_interspersed_args() options, args = parser.parse_args() @@ -190,15 +200,15 @@ def main(): runAllTests(jsontest_executable_path, input_path, use_valgrind=options.valgrind, with_json_checker=options.with_json_checker, - writerClass='StyledWriter') + writerClass='StyledWriter', output_path=options.output_path) runAllTests(jsontest_executable_path, input_path, use_valgrind=options.valgrind, with_json_checker=options.with_json_checker, - writerClass='StyledStreamWriter') + writerClass='StyledStreamWriter', output_path=options.output_path) runAllTests(jsontest_executable_path, input_path, use_valgrind=options.valgrind, with_json_checker=options.with_json_checker, - writerClass='BuiltStyledStreamWriter') + writerClass='BuiltStyledStreamWriter', output_path=options.output_path) if __name__ == '__main__': try: