1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
|
#!/usr/bin/env python3
# Copyright Catch2 Authors
# Distributed under the Boost Software License, Version 1.0.
# (See accompanying file LICENSE.txt or copy at
# https://www.boost.org/LICENSE_1_0.txt)
# SPDX-License-Identifier: BSL-1.0
import os
import subprocess
import sys
import re
import json
from collections import namedtuple
from typing import List
TestInfo = namedtuple('TestInfo', ['name', 'tags'])
cmake_version_regex = re.compile('cmake version (\d+)\.(\d+)\.(\d+)')
def get_cmake_version():
result = subprocess.run(['cmake', '--version'],
capture_output = True,
check = True,
text = True)
version_match = cmake_version_regex.match(result.stdout)
if not version_match:
print('Could not find cmake version in output')
print(f"output: '{result.stdout}'")
exit(4)
return (int(version_match.group(1)),
int(version_match.group(2)),
int(version_match.group(3)))
def build_project(sources_dir, output_base_path, catch2_path):
build_dir = os.path.join(output_base_path, 'ctest-registration-test')
config_cmd = ['cmake',
'-B', build_dir,
'-S', sources_dir,
f'-DCATCH2_PATH={catch2_path}',
'-DCMAKE_BUILD_TYPE=Debug']
build_cmd = ['cmake',
'--build', build_dir,
'--config', 'Debug']
try:
subprocess.run(config_cmd,
capture_output = True,
check = True,
text = True)
subprocess.run(build_cmd,
capture_output = True,
check = True,
text = True)
except subprocess.CalledProcessError as err:
print('Error when building the test project')
print(f'cmd: {err.cmd}')
print(f'stderr: {err.stderr}')
print(f'stdout: {err.stdout}')
exit(3)
return build_dir
def get_test_names(build_path: str) -> List[TestInfo]:
# For now we assume that Windows builds are done using MSBuild under
# Debug configuration. This means that we need to add "Debug" folder
# to the path when constructing it. On Linux, we don't add anything.
config_path = "Debug" if os.name == 'nt' else ""
full_path = os.path.join(build_path, config_path, 'tests')
cmd = [full_path, '--reporter', 'json', '--list-tests']
result = subprocess.run(cmd,
capture_output = True,
check = True,
text = True)
test_listing = json.loads(result.stdout)
assert test_listing['version'] == 1
tests = []
for test in test_listing['listings']['tests']:
test_name = test['name']
tags = test['tags']
tests.append(TestInfo(test_name, tags))
return tests
def get_ctest_listing(build_path):
old_path = os.getcwd()
os.chdir(build_path)
cmd = ['ctest', '-C', 'debug', '--show-only=json-v1']
result = subprocess.run(cmd,
capture_output = True,
check = True,
text = True)
os.chdir(old_path)
return result.stdout
def extract_tests_from_ctest(ctest_output) -> List[TestInfo]:
ctest_response = json.loads(ctest_output)
tests = ctest_response['tests']
test_infos = []
for test in tests:
test_command = test['command']
# First part of the command is the binary, second is the filter.
# If there are less, registration has failed. If there are more,
# registration has changed and the script needs updating.
assert len(test_command) == 2
test_name = test_command[1]
labels = []
for prop in test['properties']:
if prop['name'] == 'LABELS':
labels = prop['value']
test_infos.append(TestInfo(test_name, labels))
return test_infos
def check_DL_PATHS(ctest_output):
ctest_response = json.loads(ctest_output)
tests = ctest_response['tests']
for test in tests:
properties = test['properties']
for property in properties:
if property['name'] == 'ENVIRONMENT_MODIFICATION':
assert len(property['value']) == 2, f"The test provides 2 arguments to DL_PATHS, but instead found {len(property['value'])}"
def escape_catch2_test_names(infos: List[TestInfo]):
escaped = []
for info in infos:
name = info.name
for char in ('\\', ',', '[', ']'):
name = name.replace(char, f"\\{char}")
escaped.append(TestInfo(name, info.tags))
return escaped
if __name__ == '__main__':
if len(sys.argv) != 3:
print(f'Usage: {sys.argv[0]} path-to-catch2-cml output-path')
exit(2)
catch2_path = sys.argv[1]
output_base_path = sys.argv[2]
sources_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
build_path = build_project(sources_dir, output_base_path, catch2_path)
catch_test_names = escape_catch2_test_names(get_test_names(build_path))
ctest_output = get_ctest_listing(build_path)
ctest_test_names = extract_tests_from_ctest(ctest_output)
mismatched = 0
for catch_test in catch_test_names:
if catch_test not in ctest_test_names:
print(f"Catch2 test '{catch_test}' not found in CTest")
mismatched += 1
for ctest_test in ctest_test_names:
if ctest_test not in catch_test_names:
print(f"CTest test '{ctest_test}' not found in Catch2")
mismatched += 1
if mismatched:
print(f"Found {mismatched} mismatched tests catch test names and ctest test commands!")
exit(1)
print(f"{len(catch_test_names)} tests matched")
cmake_version = get_cmake_version()
if cmake_version >= (3, 27):
check_DL_PATHS(ctest_output)
|
