Keep CMake reader/writer test artifacts in the build tree

CONTRIBUTING.md

@@ -94,7 +94,11 @@ See the examples `test_complex_01.json` and `test_complex_01.expected` to better
 
 ## Understanding reader/writer test output
 
-When a test is run, output files are generated beside the input test files. Below is a short description of the content of each file:
+When running tests manually, output files are generated beside the input test files by default.
+Pass `--output-dir <directory>` to `runjsontests.py` to keep copied inputs and generated output
+in another directory. Expected results are still read from the original input directory.
+CMake uses this option to keep test artifacts in the build tree, so the source tree can be read-only.
+Below is a short description of the content of each file:
 
 * `test_complex_01.json`: input JSON document.
 * `test_complex_01.expected`: flattened JSON element tree used to check if

src/jsontestrunner/CMakeLists.txt

@@ -30,26 +30,27 @@ if(PYTHONINTERP_FOUND)
     # Run end to end parser/writer tests
     set(TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../test)
     set(RUNJSONTESTS_PATH ${TEST_DIR}/runjsontests.py)
+    set(TEST_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/test-output")
 
     # Run unit tests in post-build
     # (default cmake workflow hides away the test result into a file, resulting in poor dev workflow?!?)
     add_custom_target(jsoncpp_readerwriter_tests
-        "${PYTHON_EXECUTABLE}" -B "${RUNJSONTESTS_PATH}" $<TARGET_FILE:jsontestrunner_exe> "${TEST_DIR}/data"
+        "${PYTHON_EXECUTABLE}" -B "${RUNJSONTESTS_PATH}" --output-dir "${TEST_OUTPUT_DIR}/jsoncpp_check" $<TARGET_FILE:jsontestrunner_exe> "${TEST_DIR}/data"
         DEPENDS jsontestrunner_exe jsoncpp_test
     )
     add_custom_target(jsoncpp_check DEPENDS jsoncpp_readerwriter_tests)
 
     ## Create tests for dashboard submission, allows easy review of CI results https://my.cdash.org/index.php?project=jsoncpp
     add_test(NAME jsoncpp_readerwriter
-        COMMAND "${PYTHON_EXECUTABLE}" -B "${RUNJSONTESTS_PATH}" $<TARGET_FILE:jsontestrunner_exe> "${TEST_DIR}/data"
-        WORKING_DIRECTORY "${TEST_DIR}/data"
+        COMMAND "${PYTHON_EXECUTABLE}" -B "${RUNJSONTESTS_PATH}" --output-dir "${TEST_OUTPUT_DIR}/ctest" $<TARGET_FILE:jsontestrunner_exe> "${TEST_DIR}/data"
+        WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
     )
     add_test(NAME jsoncpp_readerwriter_json_checker
-        COMMAND "${PYTHON_EXECUTABLE}" -B "${RUNJSONTESTS_PATH}" --with-json-checker  $<TARGET_FILE:jsontestrunner_exe> "${TEST_DIR}/data"
-        WORKING_DIRECTORY "${TEST_DIR}/data"
+        COMMAND "${PYTHON_EXECUTABLE}" -B "${RUNJSONTESTS_PATH}" --with-json-checker --output-dir "${TEST_OUTPUT_DIR}/ctest" $<TARGET_FILE:jsontestrunner_exe> "${TEST_DIR}/data"
+        WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
     )
 
-    # Both tests write .actual/.actual-rewrite along with test data, need to prevent collision when running tests via ctest -j
+    # Both tests share output files, need to prevent collision when running tests via ctest -j
     set_tests_properties(jsoncpp_readerwriter jsoncpp_readerwriter_json_checker
         PROPERTIES RESOURCE_LOCK "test_data_files"
     )

test/runjsontests.py

@@ -11,6 +11,7 @@
 import os
 import os.path
 import optparse
+import shutil
 
 VALGRIND_CMD = 'valgrind --tool=memcheck --leak-check=yes --undef-value-errors=yes '
 
@@ -68,9 +69,11 @@ def __init__(self, msg):
 
 def runAllTests(jsontest_executable_path, input_path = None,
                  use_valgrind=False, with_json_checker=False,
-                 writerClass='StyledWriter'):
+                 writerClass='StyledWriter', output_path=None):
     if not input_path:
         input_path = os.path.join(os.getcwd(), 'data')
+    if output_path:
+        os.makedirs(output_path, exist_ok=True)
 
     if os.path.isdir(input_path):
         tests = [
@@ -113,12 +116,17 @@ def runAllTests(jsontest_executable_path, input_path = None,
         is_json_checker_test = os.path.basename(os.path.dirname(input_path)) == "jsonchecker"
         is_parse_only = is_json_checker_test or expect_failure
         is_strict_test = ('_strict_' in os.path.basename(input_path)) or is_json_checker_test
+        run_input_path = input_path
+        if output_path and not is_parse_only:
+            run_input_path = os.path.join(output_path, os.path.basename(input_path))
+            if not os.path.exists(run_input_path) or not os.path.samefile(input_path, run_input_path):
+                shutil.copyfile(input_path, run_input_path)
         print('TESTING:', input_path, end=' ')
         options = is_parse_only and '--parse-only' or ''
         options += is_strict_test and ' --strict' or ''
         options += ' --json-writer %s'%writerClass
         cmd = '%s%s %s "%s"' % (            valgrind_path, jsontest_executable_path, options,
-            input_path)
+            run_input_path)
         status, process_output = getStatusOutput(cmd)
         if is_parse_only:
             if expect_failure:
@@ -135,7 +143,7 @@ def runAllTests(jsontest_executable_path, input_path = None,
                 else:
                     print('OK')
         else:
-            base_path = os.path.splitext(input_path)[0]
+            base_path = os.path.splitext(run_input_path)[0]
             actual_output = safeReadFile(base_path + '.actual')
             actual_rewrite_output = safeReadFile(base_path + '.actual-rewrite')
             open(base_path + '.process-output', 'wt', encoding = 'utf-8').write(process_output)
@@ -175,6 +183,8 @@ def main():
     parser.add_option("-c", "--with-json-checker",
                   action="store_true", dest="with_json_checker", default=False,
                   help="run all the tests from the official JSONChecker test suite of json.org")
+    parser.add_option("--output-dir", dest="output_path",
+                  help="write reader/writer test artifacts here instead of beside the input files")
     parser.enable_interspersed_args()
     options, args = parser.parse_args()
 
@@ -190,15 +200,15 @@ def main():
     runAllTests(jsontest_executable_path, input_path,
                          use_valgrind=options.valgrind,
                          with_json_checker=options.with_json_checker,
-                         writerClass='StyledWriter')
+                         writerClass='StyledWriter', output_path=options.output_path)
     runAllTests(jsontest_executable_path, input_path,
                          use_valgrind=options.valgrind,
                          with_json_checker=options.with_json_checker,
-                         writerClass='StyledStreamWriter')
+                         writerClass='StyledStreamWriter', output_path=options.output_path)
     runAllTests(jsontest_executable_path, input_path,
                          use_valgrind=options.valgrind,
                          with_json_checker=options.with_json_checker,
-                         writerClass='BuiltStyledStreamWriter')
+                         writerClass='BuiltStyledStreamWriter', output_path=options.output_path)
 
 if __name__ == '__main__':
     try: