3 files changed, 227 insertions, 0 deletions
diff --git a/tests/TestScripts/DiscoverTests/CMakeLists.txt b/tests/TestScripts/DiscoverTests/CMakeLists.txt
new file mode 100644
index 0000000..f0b49f4
--- /dev/null
+++ b/tests/TestScripts/DiscoverTests/CMakeLists.txt
@@ -0,0 +1,29 @@
+cmake_minimum_required(VERSION 3.16)
+
+project(discover-tests-test
+  LANGUAGES CXX
+)
+
+add_executable(tests
+  register-tests.cpp
+)
+
+add_subdirectory(${CATCH2_PATH} catch2-build)
+target_link_libraries(tests PRIVATE Catch2::Catch2WithMain)
+
+enable_testing()
+include(Catch)
+set(extra_args)
+if (CMAKE_VERSION GREATER_EQUAL 3.27)
+    list(APPEND extra_args
+            DL_PATHS "${CMAKE_CURRENT_LIST_DIR};${CMAKE_CURRENT_LIST_DIR}/.."
+    )
+endif ()
+catch_discover_tests(
+  tests
+  ADD_TAGS_AS_LABELS
+  DISCOVERY_MODE PRE_TEST
+  ${extra_args}
+)
+
+# DISCOVERY_MODE <POST_BUILD|PRE_TEST>
diff --git a/tests/TestScripts/DiscoverTests/VerifyRegistration.py b/tests/TestScripts/DiscoverTests/VerifyRegistration.py
new file mode 100644
index 0000000..33150ca
--- /dev/null
+++ b/tests/TestScripts/DiscoverTests/VerifyRegistration.py
@@ -0,0 +1,175 @@
+#!/usr/bin/env python3
+
+#              Copyright Catch2 Authors
+# Distributed under the Boost Software License, Version 1.0.
+#   (See accompanying file LICENSE.txt or copy at
+#        https://www.boost.org/LICENSE_1_0.txt)
+
+# SPDX-License-Identifier: BSL-1.0
+
+import os
+import subprocess
+import sys
+import re
+import json
+from collections import namedtuple
+from typing import List
+
+TestInfo = namedtuple('TestInfo', ['name', 'tags'])
+
+cmake_version_regex = re.compile('cmake version (\d+)\.(\d+)\.(\d+)')
+
+def get_cmake_version():
+    result = subprocess.run(['cmake', '--version'],
+                            capture_output = True,
+                            check = True,
+                            text = True)
+    version_match = cmake_version_regex.match(result.stdout)
+    if not version_match:
+        print('Could not find cmake version in output')
+        print(f"output: '{result.stdout}'")
+        exit(4)
+    return (int(version_match.group(1)),
+            int(version_match.group(2)),
+            int(version_match.group(3)))
+
+def build_project(sources_dir, output_base_path, catch2_path):
+    build_dir = os.path.join(output_base_path, 'ctest-registration-test')
+    config_cmd = ['cmake',
+                  '-B', build_dir,
+                  '-S', sources_dir,
+                  f'-DCATCH2_PATH={catch2_path}',
+                  '-DCMAKE_BUILD_TYPE=Debug']
+
+    build_cmd = ['cmake',
+                 '--build', build_dir,
+                 '--config', 'Debug']
+
+    try:
+        subprocess.run(config_cmd,
+                       capture_output = True,
+                       check = True,
+                       text = True)
+        subprocess.run(build_cmd,
+                       capture_output = True,
+                       check = True,
+                       text = True)
+    except subprocess.CalledProcessError as err:
+        print('Error when building the test project')
+        print(f'cmd: {err.cmd}')
+        print(f'stderr: {err.stderr}')
+        print(f'stdout: {err.stdout}')
+        exit(3)
+
+    return build_dir
+
+
+
+def get_test_names(build_path: str) -> List[TestInfo]:
+    # For now we assume that Windows builds are done using MSBuild under
+    # Debug configuration. This means that we need to add "Debug" folder
+    # to the path when constructing it. On Linux, we don't add anything.
+    config_path = "Debug" if os.name == 'nt' else ""
+    full_path = os.path.join(build_path, config_path, 'tests')
+
+
+    cmd = [full_path, '--reporter', 'json', '--list-tests']
+    result = subprocess.run(cmd,
+                            capture_output = True,
+                            check = True,
+                            text = True)
+
+    test_listing = json.loads(result.stdout)
+
+    assert test_listing['version'] == 1
+
+    tests = []
+    for test in test_listing['listings']['tests']:
+        test_name = test['name']
+        tags = test['tags']
+        tests.append(TestInfo(test_name, tags))
+
+    return tests
+
+def get_ctest_listing(build_path):
+    old_path = os.getcwd()
+    os.chdir(build_path)
+
+    cmd = ['ctest', '-C', 'debug', '--show-only=json-v1']
+    result = subprocess.run(cmd,
+                            capture_output = True,
+                            check = True,
+                            text = True)
+    os.chdir(old_path)
+    return result.stdout
+
+def extract_tests_from_ctest(ctest_output) -> List[TestInfo]:
+    ctest_response = json.loads(ctest_output)
+    tests = ctest_response['tests']
+    test_infos = []
+    for test in tests:
+        test_command = test['command']
+        # First part of the command is the binary, second is the filter.
+        # If there are less, registration has failed. If there are more,
+        # registration has changed and the script needs updating.
+        assert len(test_command) == 2
+        test_name = test_command[1]
+        labels = []
+        for prop in test['properties']:
+            if prop['name'] == 'LABELS':
+                labels = prop['value']
+
+        test_infos.append(TestInfo(test_name, labels))
+
+    return test_infos
+
+def check_DL_PATHS(ctest_output):
+    ctest_response = json.loads(ctest_output)
+    tests = ctest_response['tests']
+    for test in tests:
+        properties = test['properties']
+        for property in properties:
+            if property['name'] == 'ENVIRONMENT_MODIFICATION':
+                assert len(property['value']) == 2, f"The test provides 2 arguments to DL_PATHS, but instead found {len(property['value'])}"
+
+def escape_catch2_test_names(infos: List[TestInfo]):
+    escaped = []
+    for info in infos:
+      name = info.name
+      for char in ('\\', ',', '[', ']'):
+          name = name.replace(char, f"\\{char}")
+      escaped.append(TestInfo(name, info.tags))
+    return escaped
+
+if __name__ == '__main__':
+    if len(sys.argv) != 3:
+        print(f'Usage: {sys.argv[0]} path-to-catch2-cml output-path')
+        exit(2)
+    catch2_path = sys.argv[1]
+    output_base_path = sys.argv[2]
+    sources_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
+
+    build_path = build_project(sources_dir, output_base_path, catch2_path)
+
+    catch_test_names = escape_catch2_test_names(get_test_names(build_path))
+    ctest_output = get_ctest_listing(build_path)
+    ctest_test_names = extract_tests_from_ctest(ctest_output)
+
+    mismatched = 0
+    for catch_test in catch_test_names:
+        if catch_test not in ctest_test_names:
+            print(f"Catch2 test '{catch_test}' not found in CTest")
+            mismatched += 1
+    for ctest_test in ctest_test_names:
+        if ctest_test not in catch_test_names:
+            print(f"CTest test '{ctest_test}' not found in Catch2")
+            mismatched += 1
+
+    if mismatched:
+        print(f"Found {mismatched} mismatched tests catch test names and ctest test commands!")
+        exit(1)
+    print(f"{len(catch_test_names)} tests matched")
+
+    cmake_version = get_cmake_version()
+    if cmake_version >= (3, 27):
+        check_DL_PATHS(ctest_output)
diff --git a/tests/TestScripts/DiscoverTests/register-tests.cpp b/tests/TestScripts/DiscoverTests/register-tests.cpp
new file mode 100644
index 0000000..be533ab
--- /dev/null
+++ b/tests/TestScripts/DiscoverTests/register-tests.cpp
@@ -0,0 +1,23 @@
+
+//              Copyright Catch2 Authors
+// Distributed under the Boost Software License, Version 1.0.
+//   (See accompanying file LICENSE.txt or copy at
+//        https://www.boost.org/LICENSE_1_0.txt)
+
+// SPDX-License-Identifier: BSL-1.0
+
+#include <catch2/catch_test_macros.hpp>
+
+TEST_CASE("@Script[C:\\EPM1A]=x;\"SCALA_ZERO:\"", "[script regressions]"){}
+TEST_CASE("Some test") {}
+TEST_CASE( "Let's have a test case with a long name. Longer. No, even longer. "
+           "Really looooooooooooong. Even longer than that. Multiple lines "
+           "worth of test name. Yep, like this." ) {}
+TEST_CASE( "And now a test case with weird tags.", "[tl;dr][tl;dw][foo,bar]" ) {}
+// Also check that we handle tests on class, which have name in output as 'class-name', not 'name'.
+class TestCaseFixture {
+public:
+    int m_a;
+};
+
+TEST_CASE_METHOD(TestCaseFixture, "A test case as method", "[tagstagstags]") {}