1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
|
# @file HostBasedUnitTestRunner.py
# Plugin to located any host-based unit tests in the output directory and execute them.
##
# Copyright (c) Microsoft Corporation.
# SPDX-License-Identifier: BSD-2-Clause-Patent
#
##
import os
import logging
import glob
import stat
import xml.etree.ElementTree
from edk2toolext.environment.plugintypes.uefi_build_plugin import IUefiBuildPlugin
from edk2toolext import edk2_logging
import edk2toollib.windows.locate_tools as locate_tools
from edk2toolext.environment import shell_environment
from edk2toollib.utility_functions import RunCmd
from edk2toollib.utility_functions import GetHostInfo
from textwrap import dedent
class HostBasedUnitTestRunner(IUefiBuildPlugin):
def do_pre_build(self, thebuilder):
'''
Run Prebuild
'''
return 0
def do_post_build(self, thebuilder):
'''
After a build, will automatically locate and run all host-based unit tests. Logs any
failures with Warning severity and will return a count of the failures as the return code.
EXPECTS:
- Build Var 'CI_BUILD_TYPE' - If not set to 'host_unit_test', will not do anything.
UPDATES:
- Shell Var 'CMOCKA_XML_FILE'
'''
ci_type = thebuilder.env.GetValue('CI_BUILD_TYPE')
if ci_type != 'host_unit_test':
return 0
shell_env = shell_environment.GetEnvironment()
logging.log(edk2_logging.get_section_level(),
"Run Host based Unit Tests")
path = thebuilder.env.GetValue("BUILD_OUTPUT_BASE")
failure_count = 0
# Set up the reporting type for Cmocka.
shell_env.set_shell_var('CMOCKA_MESSAGE_OUTPUT', 'xml')
for arch in thebuilder.env.GetValue("TARGET_ARCH").split():
logging.log(edk2_logging.get_subsection_level(),
"Testing for architecture: " + arch)
cp = os.path.join(path, arch)
# If any old results XML files exist, clean them up.
for old_result in glob.iglob(os.path.join(cp, "*.result.xml")):
os.remove(old_result)
# Find and Run any Host Tests
if GetHostInfo().os.upper() == "LINUX":
testList = glob.glob(os.path.join(cp, "*Test*"))
for a in testList[:]:
p = os.path.join(cp, a)
# It must be a file
if not os.path.isfile(p):
testList.remove(a)
logging.debug(f"Remove directory file: {p}")
continue
# It must be executable
if os.stat(p).st_mode & (stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) == 0:
testList.remove(a)
logging.debug(f"Remove non-executable file: {p}")
continue
logging.info(f"Test file found: {p}")
elif GetHostInfo().os.upper() == "WINDOWS":
testList = glob.glob(os.path.join(cp, "*Test*.exe"))
else:
raise NotImplementedError("Unsupported Operating System")
if not testList:
logging.warning(dedent("""
UnitTest Coverage:
No unit tests discovered. Test coverage will not be generated.
Prevent this message by:
1. Adding host-based unit tests to this package
2. Ensuring tests have the word "Test" in their name
3. Disabling HostUnitTestCompilerPlugin in the package CI YAML file
""").strip())
return 0
for test in testList:
# Configure output name if test uses cmocka.
shell_env.set_shell_var(
'CMOCKA_XML_FILE', test + ".CMOCKA.%g." + arch + ".result.xml")
# Configure output name if test uses gtest.
shell_env.set_shell_var(
'GTEST_OUTPUT', "xml:" + test + ".GTEST." + arch + ".result.xml")
# Run the test.
ret = RunCmd('"' + test + '"', "", workingdir=cp)
if ret != 0:
logging.error("UnitTest Execution Error: " +
os.path.basename(test))
else:
logging.info("UnitTest Completed: " +
os.path.basename(test))
file_match_pattern = test + ".*." + arch + ".result.xml"
xml_results_list = glob.glob(file_match_pattern)
for xml_result_file in xml_results_list:
root = xml.etree.ElementTree.parse(
xml_result_file).getroot()
for suite in root:
for case in suite:
for result in case:
if result.tag == 'failure':
logging.warning(
"%s Test Failed" % os.path.basename(test))
logging.warning(
" %s - %s" % (case.attrib['name'], result.text))
failure_count += 1
if thebuilder.env.GetValue("CODE_COVERAGE") != "FALSE":
if thebuilder.env.GetValue("TOOL_CHAIN_TAG") == "GCC5":
ret = self.gen_code_coverage_gcc(thebuilder)
if ret != 0:
failure_count += 1
elif thebuilder.env.GetValue("TOOL_CHAIN_TAG").startswith ("VS"):
ret = self.gen_code_coverage_msvc(thebuilder)
if ret != 0:
failure_count += 1
else:
logging.info("Skipping code coverage. Currently, support GCC and MSVC compiler.")
return failure_count
def gen_code_coverage_gcc(self, thebuilder):
logging.info("Generating UnitTest code coverage")
buildOutputBase = thebuilder.env.GetValue("BUILD_OUTPUT_BASE")
workspace = thebuilder.env.GetValue("WORKSPACE")
# Generate base code coverage for all source files
ret = RunCmd("lcov", f"--no-external --capture --initial --directory {buildOutputBase} --output-file {buildOutputBase}/cov-base.info --rc lcov_branch_coverage=1")
if ret != 0:
logging.error("UnitTest Coverage: Failed to build initial coverage data.")
return 1
# Coverage data for tested files only
ret = RunCmd("lcov", f"--capture --directory {buildOutputBase}/ --output-file {buildOutputBase}/coverage-test.info --rc lcov_branch_coverage=1")
if ret != 0:
logging.error("UnitTest Coverage: Failed to build coverage data for tested files.")
return 1
# Aggregate all coverage data
ret = RunCmd("lcov", f"--add-tracefile {buildOutputBase}/cov-base.info --add-tracefile {buildOutputBase}/coverage-test.info --output-file {buildOutputBase}/total-coverage.info --rc lcov_branch_coverage=1")
if ret != 0:
logging.error("UnitTest Coverage: Failed to aggregate coverage data.")
return 1
# Generate coverage XML
ret = RunCmd("lcov_cobertura",f"{buildOutputBase}/total-coverage.info -o {buildOutputBase}/compare.xml")
if ret != 0:
logging.error("UnitTest Coverage: Failed to generate coverage XML.")
return 1
# Filter out auto-generated and test code
ret = RunCmd("lcov_cobertura",f"{buildOutputBase}/total-coverage.info --excludes ^.*UnitTest\|^.*MU\|^.*Mock\|^.*DEBUG -o {buildOutputBase}/coverage.xml")
if ret != 0:
logging.error("UnitTest Coverage: Failed generate filtered coverage XML.")
return 1
# Generate all coverage file
testCoverageList = glob.glob (f"{workspace}/Build/**/total-coverage.info", recursive=True)
coverageFile = ""
for testCoverage in testCoverageList:
coverageFile += " --add-tracefile " + testCoverage
ret = RunCmd("lcov", f"{coverageFile} --output-file {workspace}/Build/all-coverage.info --rc lcov_branch_coverage=1")
if ret != 0:
logging.error("UnitTest Coverage: Failed generate all coverage file.")
return 1
# Generate and XML file if requested.for all package
if os.path.isfile(f"{workspace}/Build/coverage.xml"):
os.remove(f"{workspace}/Build/coverage.xml")
ret = RunCmd("lcov_cobertura",f"{workspace}/Build/all-coverage.info --excludes ^.*UnitTest\|^.*MU\|^.*Mock\|^.*DEBUG -o {workspace}/Build/coverage.xml")
return 0
def gen_code_coverage_msvc(self, thebuilder):
logging.info("Generating UnitTest code coverage")
buildOutputBase = thebuilder.env.GetValue("BUILD_OUTPUT_BASE")
testList = glob.glob(os.path.join(buildOutputBase, "**","*Test*.exe"), recursive=True)
workspace = thebuilder.env.GetValue("WORKSPACE")
workspace = (workspace + os.sep) if workspace[-1] != os.sep else workspace
workspaceBuild = os.path.join(workspace, 'Build')
# Generate coverage file
coverageFile = ""
for testFile in testList:
ret = RunCmd("OpenCppCoverage", f"--source {workspace} --export_type binary:{testFile}.cov -- {testFile}")
if ret != 0:
logging.error("UnitTest Coverage: Failed to collect coverage data.")
return 1
coverageFile = f" --input_coverage={testFile}.cov"
totalCoverageFile = os.path.join(buildOutputBase, 'coverage.cov')
if os.path.isfile(totalCoverageFile):
coverageFile += f" --input_coverage={totalCoverageFile}"
ret = RunCmd(
"OpenCppCoverage",
f"--export_type binary:{totalCoverageFile} " +
f"--working_dir={workspaceBuild} " +
f"{coverageFile}"
)
if ret != 0:
logging.error("UnitTest Coverage: Failed to collect coverage data.")
return 1
# Generate and XML file if requested.by each package
ret = RunCmd(
"OpenCppCoverage",
f"--export_type cobertura:{os.path.join(buildOutputBase, 'coverage.xml')} " +
f"--working_dir={workspaceBuild} " +
f"--input_coverage={totalCoverageFile} "
)
if ret != 0:
logging.error("UnitTest Coverage: Failed to generate cobertura format xml in single package.")
return 1
# Generate total report XML file for all package
testCoverageList = glob.glob(os.path.join(workspace, "Build", "**", "*Test*.exe.cov"), recursive=True)
coverageFile = ""
totalCoverageFile = os.path.join(workspaceBuild, 'coverage.cov')
for testCoverage in testCoverageList:
coverageFile = f" --input_coverage={testCoverage}"
if os.path.isfile(totalCoverageFile):
coverageFile += f" --input_coverage={totalCoverageFile}"
ret = RunCmd(
"OpenCppCoverage",
f"--export_type binary:{totalCoverageFile} " +
f"--working_dir={workspaceBuild} " +
f"{coverageFile}"
)
if ret != 0:
logging.error("UnitTest Coverage: Failed to collect coverage data.")
return 1
ret = RunCmd(
"OpenCppCoverage",
f"--export_type cobertura:{os.path.join(workspaceBuild, 'coverage.xml')} " +
f"--working_dir={workspaceBuild} " +
f"--input_coverage={totalCoverageFile}"
)
if ret != 0:
logging.error("UnitTest Coverage: Failed to generate cobertura format xml.")
return 1
return 0
|
