From de6cbac3c4e44de797bc2a4755fb765cf5a49f55 Mon Sep 17 00:00:00 2001 From: Paul Fagerburg Date: Tue, 11 May 2021 09:56:48 -0600 Subject: tests: improve code coverage support Fix the exclusion path for lcov; it should exclude the directory with source code, not object files. Use the COV environment variable to * control whether we build for coverage or not * select the output directory Add a separate target for generating the report, so we can get a report for all of the tests together or just a single test. Add documentation. Signed-off-by: Paul Fagerburg Change-Id: I2bd2bfdedfab291aabeaa968c10b17e9b61c9c0a Reviewed-on: https://review.coreboot.org/c/coreboot/+/54072 Tested-by: build bot (Jenkins) Reviewed-by: Jakub Czapiga --- Documentation/technotes/2021-05-code-coverage.md | 56 ++++++++++++++++++++++++ Documentation/tutorial/part3.md | 2 + Makefile | 4 +- tests/Makefile.inc | 56 +++++++++++++++++++----- 4 files changed, 104 insertions(+), 14 deletions(-) create mode 100644 Documentation/technotes/2021-05-code-coverage.md diff --git a/Documentation/technotes/2021-05-code-coverage.md b/Documentation/technotes/2021-05-code-coverage.md new file mode 100644 index 000000000000..ec7f4cb5f979 --- /dev/null +++ b/Documentation/technotes/2021-05-code-coverage.md @@ -0,0 +1,56 @@ +# Unit Test Code Coverage + +Code coverage for the coreboot unit tests allows us to see what lines of +code in the coreboot library are covered by unit tests, and allows a test +author to see where they need to add test cases for additional coverage. + +Enable code coverage in your unit test build by setting the environment +variable `COV` to 1; either `export COV=1` in your shell, or add it to your +`make` command, e.g. `COV=1 make unit-tests`. + +The build output directory is either `build/tests` or `build/coverage`, +depending on whether `COV=1` is set in the environment. + +All of the unit test targets are available with and without `COV=1` +* `clean-unit-tests` +* `build-unit-tests` +* `run-unit-tests` +* `unit-tests` (which is just `build-unit-tests` followed by `run-unit-tests`) + +There are two new `make` targets: +* `coverage-report` generates a code coverage report from all of the +GCOV data (`*.gcda` and `*.gcno` files) in the build directory. To view the +coverage report, open `build/coverage/coverage_reports/index.html` in your web +browser. +* `clean-coverage-report` deletes just the coverage report. + +The `coverage-report` and `clean-coverage-report` targets automatically set +`COV=1` if it is not already set in the environment. + + +## Examples + +`COV=1 make unit-tests coverage-report` builds all of the unit tests with code +coverage, runs the unit tests, and generates the code coverage report. + +`COV=1 make build-unit-tests` builds all of the unit tests with code coverage. + +`COV=1 make run-unit-tests` runs the unit tests, building them with code +coverage if they are out-of-date. + +`COV=1 make coverage-report` creates the code coverage report. This +target does not explicitly depend on the tests being built and run; it gathers +the code coverage data from the output directory, which it assumes already +exists. + +`COV=1 make tests/lib/uuid-test coverage-report` builds the uuid test +with code coverage, runs it, and generates a code coverage report just for +that test. + +As a demonstration that building with and without coverage uses different +output directories: +1. `make build-unit-tests` builds unit tests without code coverage into +`build/tests`. +2. `COV=1 make clean-unit-tests` cleans `build/coverage` +3. `make build-unit-tests` doesn't need to build anything in `build/tests`, +because those files weren't affected by the previous `clean-unit-tests`. diff --git a/Documentation/tutorial/part3.md b/Documentation/tutorial/part3.md index 7ccee877546e..6d147ff9a9e6 100644 --- a/Documentation/tutorial/part3.md +++ b/Documentation/tutorial/part3.md @@ -3,6 +3,8 @@ ## Introduction General thoughts about unit testing coreboot can be found in [Unit testing coreboot](../technotes/2020-03-unit-testing-coreboot.md). +Additionally, [code coverage](../technotes/2021-05-code-coverage.md) support +is available for unit tests. This document aims to guide developers through the process of adding and writing unit tests for coreboot modules. diff --git a/Makefile b/Makefile index e25c081ac899..ea196c961a6c 100644 --- a/Makefile +++ b/Makefile @@ -124,8 +124,8 @@ ifneq ($(filter help%, $(MAKECMDGOALS)), ) NOCOMPILE:=1 UNIT_TEST:=1 else -ifneq ($(filter %-test %-tests, $(MAKECMDGOALS)),) -ifneq ($(filter-out %-test %-tests, $(MAKECMDGOALS)),) +ifneq ($(filter %-test %-tests %coverage-report, $(MAKECMDGOALS)),) +ifneq ($(filter-out %-test %-tests %coverage-report, $(MAKECMDGOALS)),) $(error Cannot mix unit-tests targets with other targets) endif UNIT_TEST:=1 diff --git a/tests/Makefile.inc b/tests/Makefile.inc index cd25e0f80998..6cf92fa999c8 100644 --- a/tests/Makefile.inc +++ b/tests/Makefile.inc @@ -1,9 +1,18 @@ # SPDX-License-Identifier: GPL-2.0-only testsrc = $(top)/tests + +# Place the build output in one of two places depending on COV, so that code +# built with code coverage never mixes with code built without code coverage. +ifeq ($(COV),1) +testobj = $(obj)/coverage +else testobj = $(obj)/tests +endif + cmockasrc = 3rdparty/cmocka cmockaobj = $(objutil)/cmocka +coverage_dir = coverage_reports CMOCKA_LIB := $(cmockaobj)/src/libcmocka.so @@ -51,6 +60,12 @@ TEST_LDFLAGS += -Wl,--gc-sections TEST_CFLAGS += -fno-pie -fno-pic TEST_LDFLAGS += -no-pie +# Enable code coverage if COV=1 +ifeq ($(COV),1) +TEST_CFLAGS += --coverage +TEST_LDFLAGS += --coverage +endif + # Extra attributes for unit tests, declared per test attributes:= srcs cflags config mocks stage @@ -99,7 +114,7 @@ $$($(1)-config-file): $(TEST_KCONFIG_AUTOHEADER) $($(1)-objs): TEST_CFLAGS += -I$$(dir $$($(1)-config-file)) \ -D__$$(shell echo $$($(1)-stage) | tr '[:lower:]' '[:upper:]')__ -$($(1)-objs): $(obj)/$(1)/%.o: $$$$*.c $$($(1)-config-file) +$($(1)-objs): $(testobj)/$(1)/%.o: $$$$*.c $$($(1)-config-file) mkdir -p $$(dir $$@) $(HOSTCC) $(HOSTCFLAGS) $$(TEST_CFLAGS) $($(1)-cflags) -MMD \ -MT $$@ -c $$< -o $$@ @@ -111,10 +126,10 @@ $($(1)-bin): $($(1)-objs) $(CMOCKA_LIB) endef $(foreach test, $(alltests), \ - $(eval $(test)-objs:=$(addprefix $(obj)/$(test)/, \ + $(eval $(test)-objs:=$(addprefix $(testobj)/$(test)/, \ $(patsubst %.c,%.o,$($(test)-srcs))))) $(foreach test, $(alltests), \ - $(eval $(test)-bin:=$(obj)/$(test)/run)) + $(eval $(test)-bin:=$(testobj)/$(test)/run)) $(foreach test, $(alltests), \ $(eval $(call TEST_CC_template,$(test)))) @@ -168,15 +183,31 @@ $(alltests): $$($$(@)-bin) rm -f $(testobj)/junit-$(subst /,_,$^).xml $(testobj)/$(subst /,_,$^).failed -./$^ || echo failed > $(testobj)/$(subst /,_,$^).failed -.PHONY: coverage-unit-tests +# Build a code coverage report by collecting all the gcov files into a single +# report. If COV is not set, this might be a user error, and they're trying +# to generate a coverage report without first having built and run the code +# with code coverage. So instead of silently correcting it by adding COV=1, +# let's flag it to the user so they can be sure they're doing the thing they +# want to do. -coverage-unit-tests: TEST_CFLAGS += --coverage -coverage-unit-tests: TEST_LDFLAGS += --coverage -coverage-unit-tests: clean-unit-tests unit-tests - lcov -o $(testobj)/tests.info -c -d $(testobj) --exclude '*/$(testobj)/*' - genhtml -q -o build/tests/coverage_rpt -t "coreboot unit tests" \ +.PHONY: coverage-report clean-coverage-report + +ifeq ($(COV),1) +coverage-report: + lcov -o $(testobj)/tests.info -c -d $(testobj) --exclude '$(testsrc)/*' + genhtml -q -o $(testobj)/$(coverage_dir) -t "coreboot unit tests" \ -s $(testobj)/tests.info +clean-coverage-report: + rm -Rf $(testobj)/$(coverage_dir) +else +coverage-report: + COV=1 V=$(V) $(MAKE) coverage-report + +clean-coverage-report: + COV=1 V=$(V) $(MAKE) clean-coverage-report +endif + unit-tests: build-unit-tests run-unit-tests build-unit-tests: $(test-bins) @@ -195,7 +226,7 @@ run-unit-tests: $(alltests) fi $(addprefix clean-,$(alltests)): clean-%: - rm -rf $(obj)/$* + rm -rf $(testobj)/$* clean-unit-tests: rm -rf $(testobj) @@ -208,11 +239,12 @@ list-unit-tests: help-unit-tests help:: @echo '*** coreboot unit-tests targets ***' + @echo ' Use "COV=1 make [target]" to enable code coverage for unit tests' @echo ' unit-tests - Run all unit-tests from tests/' @echo ' clean-unit-tests - Remove unit-tests build artifacts' @echo ' list-unit-tests - List all unit-tests' @echo ' - Build and run single unit-test' @echo ' clean- - Remove single unit-test build artifacts' - @echo ' coverage-unit-tests - Build unit tests for code coverage and' - @echo ' generate a code coverage report' + @echo ' coverage-report - Generate a code coverage report' + @echo ' clean-coverage-report - Remove the code coverage report' @echo -- cgit v1.2.3