Merge branch 'instrumentation-query-schema' into 'master'

instrumentation: Add JSON schema for query files

See merge request cmake/cmake!11600

Help/manual/cmake-instrumentation.7.rst

@@ -265,6 +265,9 @@ be present in all snippet files. User written ``callbacks`` should be able to
 handle the presence of this optional data, since it may be requested by an
 unrelated query.
 
+The JSON format is described in machine-readable form by
+:download:`this JSON schema </manual/instrumentation/query-v1-schema.json>`.
+
 Example:
 
 .. code-block:: json

Help/manual/instrumentation/query-v1-schema.json

@@ -0,0 +1,54 @@
+{
+  "$schema": "http://json-schema.org/2020-12/schema#",
+  "type": "object",
+  "required": ["version"],
+  "properties": {
+    "version": {
+      "type": "integer",
+      "description": "The data version of snippet file to generate.",
+      "enum": [
+        1
+      ]
+    },
+    "callbacks": {
+      "type": "array",
+      "description": "A list of command-line strings for callbacks to handle collected instrumentation data. Whenever these callbacks are executed, the full path to a v1 Index File is appended to the arguments included in the string.",
+      "items": {
+        "type": "string"
+      }
+    },
+    "hooks": {
+      "type": "array",
+      "description": "A list of strings specifying when indexing should occur automatically. These are the intervals when instrumentation data should be collated and user callbacks should be invoked to handle the data.",
+      "items": {
+        "enum": [
+          "postGenerate",
+          "preBuild",
+          "postBuild",
+          "preCMakeBuild",
+          "postCMakeBuild",
+          "postCMakeInstall",
+          "postCMakeWorkflow",
+          "postCTest"
+        ],
+        "type": "string"
+      },
+      "uniqueItems": true
+    },
+    "options": {
+      "type": "array",
+      "description": "A list of strings used to enable certain optional behavior, including the collection of certain additional data.",
+      "uniqueItems": true,
+      "items": {
+        "enum": [
+          "staticSystemInformation",
+          "dynamicSystemInformation",
+          "cdashSubmit",
+          "cdashVerbose",
+          "trace"
+        ],
+        "type": "string"
+      }
+    }
+  }
+}

Tests/RunCMake/CMakeLists.txt

@@ -428,7 +428,8 @@ add_RunCMake_test(FileAPI -DPython_EXECUTABLE=${Python_EXECUTABLE}
                           -DCMAKE_CXX_SIMULATE_ID=${CMAKE_CXX_SIMULATE_ID}
                           -DCMake_TEST_JSON_SCHEMA=${CMake_TEST_JSON_SCHEMA})
 if(CMAKE_GENERATOR MATCHES "Make|Ninja|FASTBuild")
-  add_RunCMake_test(Instrumentation)
+  add_RunCMake_test(Instrumentation -DPython_EXECUTABLE=${Python_EXECUTABLE}
+                                    -DCMake_TEST_JSON_SCHEMA=${CMake_TEST_JSON_SCHEMA})
 endif()
 add_RunCMake_test(ConfigDir)
 if(CMake_TEST_FindPython2)

Tests/RunCMake/Instrumentation/RunCMakeTest.cmake

@@ -182,9 +182,15 @@ instrument(empty)
 instrument(bad-version)
 
 # Verify Hooks Run and Index File
-instrument(hooks-1 BUILD INSTALL TEST STATIC_QUERY)
-instrument(hooks-2 BUILD INSTALL TEST)
-instrument(hooks-no-callbacks MANUAL_HOOK)
+instrument(hooks-1 BUILD INSTALL TEST STATIC_QUERY
+  CHECK_SCRIPT check-query-dir.cmake
+)
+instrument(hooks-2 BUILD INSTALL TEST
+  CHECK_SCRIPT check-query-dir.cmake
+)
+instrument(hooks-no-callbacks MANUAL_HOOK
+  CHECK_SCRIPT check-query-dir.cmake
+)
 
 # Check data file contents for optional query data
 instrument(no-query

Tests/RunCMake/Instrumentation/check-data-dir.cmake

@@ -1,5 +1,6 @@
 include(${CMAKE_CURRENT_LIST_DIR}/verify-snippet.cmake)
 include(${CMAKE_CURRENT_LIST_DIR}/json.cmake)
+include(${CMAKE_CURRENT_LIST_DIR}/check-query-dir.cmake)
 
 file(GLOB snippets LIST_DIRECTORIES false ${v1}/data/*)
 if (NOT snippets)

Tests/RunCMake/Instrumentation/check-query-dir.cmake

@@ -0,0 +1,8 @@
+include(${CMAKE_CURRENT_LIST_DIR}/verify-snippet.cmake)
+include(${CMAKE_CURRENT_LIST_DIR}/json.cmake)
+include(${CMAKE_CURRENT_LIST_DIR}/validate_schema.cmake)
+file(GLOB_RECURSE queries LIST_DIRECTORIES false ${v1}/query/*)
+
+foreach(query ${queries})
+  validate_schema("${query}" "${CMAKE_CURRENT_LIST_DIR}/../../../Help/manual/instrumentation/query-v1-schema.json")
+endforeach()

Tests/RunCMake/Instrumentation/json.cmake

@@ -1,3 +1,5 @@
+include_guard()
+
 # Read the JSON `filename` into `outvar`.
 function(read_json filename outvar)
   file(READ "${filename}" ${outvar})

Tests/RunCMake/Instrumentation/validate_schema.cmake

@@ -0,0 +1,15 @@
+function(validate_schema json_file schema_file)
+  if (Python_EXECUTABLE AND CMake_TEST_JSON_SCHEMA)
+    execute_process(
+      COMMAND ${Python_EXECUTABLE} "${CMAKE_CURRENT_LIST_DIR}/validate_schema.py" "${json_file}" "${schema_file}"
+      RESULT_VARIABLE result
+      OUTPUT_VARIABLE output
+      ERROR_VARIABLE output
+    )
+    if(NOT result STREQUAL 0)
+      string(REPLACE "\n" "\n  " output "${output}")
+      string(APPEND RunCMake_TEST_FAILED "Failed to validate version JSON schema for file: ${json_file}\nOutput:\n${output}\n")
+    endif()
+    return(PROPAGATE RunCMake_TEST_FAILED)
+  endif()
+endfunction()

Tests/RunCMake/Instrumentation/validate_schema.py

@@ -0,0 +1,15 @@
+import json
+import jsonschema
+import sys
+
+with open(sys.argv[1], "r", encoding="utf-8-sig") as f:
+    contents = json.load(f)
+
+with open(sys.argv[2], "r", encoding="utf-8") as f:
+    schema = json.load(f)
+
+try:
+    jsonschema.validate(contents, schema)
+except jsonschema.ValidationError as e:
+    print(e)
+    sys.exit(1)

Tests/RunCMake/Instrumentation/verify-snippet.cmake

@@ -1,4 +1,5 @@
 # Performs generic (non-project specific) validation of v1 Snippet File Contents
+include_guard()
 
 include(${CMAKE_CURRENT_LIST_DIR}/json.cmake)