mirror of
https://github.com/apple/container.git
synced 2026-08-24 02:24:19 -05:00
Add unit and integration coverage report make targets (#1427)
This adds make targets that generate json, html, and plain text coverage reports for unit tests, integration tests, and an aggregation of both suites. These new targets can be run via `make coverage` (aggregate), `make coverage-unit` (only unit tests), and `make coverage-integration` (only integration tests). ## Type of Change - [ ] Bug fix - [X] New feature - [ ] Breaking change - [ ] Documentation update ## Motivation and Context This provides developers a means to measure test coverage for new and old features alike, to ensure that testing for aforementinoed features is adequate. ## Testing - [X] Tested locally - [ ] Added/updated tests - [ ] Added/updated docs
This commit is contained in:
Noah Thornton
@@ -24,6 +24,10 @@ test_results/
|
||||
*.pkg
|
||||
*.swp
|
||||
|
||||
# Coverage artifacts
|
||||
*.profraw
|
||||
coverage-reports/
|
||||
|
||||
# API docs for local preview only.
|
||||
_site/
|
||||
_serve/
|
||||
|
||||
@@ -24,14 +24,13 @@ SWIFT := "/usr/bin/swift"
|
||||
DEST_DIR ?= /usr/local/
|
||||
ROOT_DIR := $(shell git rev-parse --show-toplevel)
|
||||
BUILD_BIN_DIR = $(shell $(SWIFT) build -c $(BUILD_CONFIGURATION) --show-bin-path)
|
||||
COV_DATA_DIR = $(shell $(SWIFT) test --show-coverage-path | xargs dirname)
|
||||
COV_REPORT_FILE = $(ROOT_DIR)/code-coverage-report
|
||||
STAGING_DIR := bin/$(BUILD_CONFIGURATION)/staging/
|
||||
PKG_PATH := bin/$(BUILD_CONFIGURATION)/container-installer-unsigned.pkg
|
||||
DSYM_DIR := bin/$(BUILD_CONFIGURATION)/bundle/container-dSYM
|
||||
DSYM_PATH := bin/$(BUILD_CONFIGURATION)/bundle/container-dSYM.zip
|
||||
CODESIGN_OPTS ?= --force --sign - --timestamp=none
|
||||
|
||||
|
||||
# Conditionally use a temporary data directory for integration tests
|
||||
SYSTEM_START_OPTS :=
|
||||
ifneq ($(strip $(APP_ROOT)),)
|
||||
@@ -154,29 +153,114 @@ install-kernel:
|
||||
@echo Starting system to install kernel
|
||||
@bin/container --debug system start --timeout 60 --enable-kernel-install $(SYSTEM_START_OPTS)
|
||||
|
||||
|
||||
# Coverage report generation helpers
|
||||
# Directory that swift test spits out raw coverage data
|
||||
COV_DATA_DIR = $(shell $(SWIFT) test --show-coverage-path | xargs dirname)
|
||||
COV_REPORT_FILE = $(ROOT_DIR)/code-coverage-report
|
||||
COVERAGE_OUTPUT_DIR := $(ROOT_DIR)/coverage-reports
|
||||
TEST_BINARY = $(BUILD_BIN_DIR)/containerPackageTests.xctest/Contents/MacOS/containerPackageTests
|
||||
# Set of files we do not want to get caught in the coverage generation
|
||||
LLVM_COV_IGNORE := \
|
||||
--ignore-filename-regex=".build/" \
|
||||
--ignore-filename-regex=".pb.swift" \
|
||||
--ignore-filename-regex=".proto" \
|
||||
--ignore-filename-regex=".grpc.swift"
|
||||
# swift test overwrites profraw data on each invocation, so we copy to a safe spot
|
||||
SAVE_PROFRAW = mkdir -p $(COVERAGE_OUTPUT_DIR)/integration/$(1) && cp $(COV_DATA_DIR)/*.profraw $(COVERAGE_OUTPUT_DIR)/integration/$(1)/
|
||||
|
||||
# Generate JSON + HTML coverage reports and a coverage-percent.txt from a profdata file.
|
||||
# $(1) = profdata path, $(2) = tier name (unit/integration/combined)
|
||||
define GENERATE_COV_REPORTS
|
||||
@echo Exporting $(2) coverage JSON...
|
||||
@xcrun llvm-cov export --compilation-dir=`pwd` \
|
||||
-instr-profile=$(1) \
|
||||
$(LLVM_COV_IGNORE) \
|
||||
$(TEST_BINARY) > $(COVERAGE_OUTPUT_DIR)/$(2)/coverage-summary.json
|
||||
@echo Generating $(2) coverage HTML report...
|
||||
@xcrun llvm-cov show --compilation-dir=`pwd` --format=html \
|
||||
-instr-profile=$(1) \
|
||||
$(LLVM_COV_IGNORE) \
|
||||
-output-dir=$(COVERAGE_OUTPUT_DIR)/$(2)/html \
|
||||
$(TEST_BINARY)
|
||||
@echo Extracting $(2) coverage percentages...
|
||||
@jq -r '"line coverage: \(.data[0].totals.lines.percent * 100 | round / 100)%\nfunction coverage: \(.data[0].totals.functions.percent * 100 | round / 100)%"' \
|
||||
$(COVERAGE_OUTPUT_DIR)/$(2)/coverage-summary.json > $(COVERAGE_OUTPUT_DIR)/$(2)/coverage-percent.txt
|
||||
@cat $(COVERAGE_OUTPUT_DIR)/$(2)/coverage-percent.txt
|
||||
endef
|
||||
|
||||
INTEGRATION_TEST_SUITES := \
|
||||
TestCLIHelp \
|
||||
TestCLIStatus \
|
||||
TestCLIVersion \
|
||||
TestCLINetwork \
|
||||
TestCLIRunLifecycle \
|
||||
TestCLIRunCapabilities \
|
||||
TestCLIExecCommand \
|
||||
TestCLICreateCommand \
|
||||
TestCLIRunCommand1 \
|
||||
TestCLIRunCommand2 \
|
||||
TestCLIRunCommand3 \
|
||||
TestCLIPruneCommand \
|
||||
TestCLIRegistry \
|
||||
TestCLIStatsCommand \
|
||||
TestCLIImagesCommand \
|
||||
TestCLIRunBase \
|
||||
TestCLIRunInitImage \
|
||||
TestCLIBuildBase \
|
||||
TestCLIExportCommand \
|
||||
TestCLIVolumes \
|
||||
TestCLIKernelSet \
|
||||
TestCLIAnonymousVolumes \
|
||||
TestCLINotFound \
|
||||
TestCLINoParallelCases
|
||||
|
||||
.PHONY: coverage
|
||||
coverage: init-block
|
||||
@echo Ensuring apiserver stopped before the coverage analysis
|
||||
# Merge the raw coverage data generated from coverage-unit and coverage-integration into one unified report
|
||||
coverage: coverage-unit coverage-integration
|
||||
@echo Merging combined coverage profdata...
|
||||
@mkdir -p $(COVERAGE_OUTPUT_DIR)/combined
|
||||
@xcrun llvm-profdata merge -sparse \
|
||||
$(COVERAGE_OUTPUT_DIR)/unit/default.profdata \
|
||||
$(COVERAGE_OUTPUT_DIR)/integration/default.profdata \
|
||||
-o $(COVERAGE_OUTPUT_DIR)/combined/default.profdata
|
||||
$(call GENERATE_COV_REPORTS,$(COVERAGE_OUTPUT_DIR)/combined/default.profdata,combined)
|
||||
|
||||
.PHONY: coverage-unit
|
||||
coverage-unit:
|
||||
@echo Running unit test coverage...
|
||||
@rm -f $(COV_DATA_DIR)/*.profraw
|
||||
@mkdir -p $(COVERAGE_OUTPUT_DIR)/unit
|
||||
# Run the test suite with profiler (exclude integration tests)
|
||||
@$(SWIFT) test --enable-code-coverage -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --skip TestCLI
|
||||
# Move the profiling data to a staging area for later consumption in full coverage aggregation
|
||||
@echo Merging unit coverage profdata...
|
||||
@xcrun llvm-profdata merge -sparse $(COV_DATA_DIR)/*.profraw -o $(COVERAGE_OUTPUT_DIR)/unit/default.profdata
|
||||
# Generate both JSON (for machines) and html (for humans)
|
||||
$(call GENERATE_COV_REPORTS,$(COVERAGE_OUTPUT_DIR)/unit/default.profdata,unit)
|
||||
|
||||
.PHONY: coverage-integration
|
||||
coverage-integration: all
|
||||
@echo Ensuring apiserver stopped before the coverage integration tests...
|
||||
@bin/container system stop && sleep 3 && scripts/ensure-container-stopped.sh
|
||||
@bin/container --debug system start $(SYSTEM_START_OPTS) && \
|
||||
echo "Starting coverage analysis" && \
|
||||
@echo Running integration test coverage...
|
||||
@rm -f $(COV_DATA_DIR)/*.profraw
|
||||
@mkdir -p $(COVERAGE_OUTPUT_DIR)/integration
|
||||
@bin/container --debug system start --timeout 60 $(SYSTEM_START_OPTS) && \
|
||||
echo "Starting CLI integration tests with coverage" && \
|
||||
{ \
|
||||
exit_code=0; \
|
||||
$(SWIFT) test --no-parallel --enable-code-coverage -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) || exit_code=1 ; \
|
||||
echo Ensuring apiserver stopped after the CLI integration tests ; \
|
||||
export CLITEST_LOG_ROOT=$(LOG_ROOT) ; \
|
||||
$(foreach suite,$(INTEGRATION_TEST_SUITES), \
|
||||
$(SWIFT) test --no-parallel --enable-code-coverage -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter $(suite) || exit_code=1 ; $(call SAVE_PROFRAW,$(suite)) ; \
|
||||
) \
|
||||
echo Ensuring apiserver stopped after the coverage integration tests ; \
|
||||
scripts/ensure-container-stopped.sh ; \
|
||||
echo Generating code coverage report... ; \
|
||||
xcrun llvm-profdata merge -sparse $(COV_DATA_DIR)/*.profraw -o $(COV_DATA_DIR)/default.profdata ; \
|
||||
xcrun llvm-cov show --compilation-dir=`pwd` \
|
||||
-instr-profile=$(COV_DATA_DIR)/default.profdata \
|
||||
--ignore-filename-regex=".build/" \
|
||||
--ignore-filename-regex=".pb.swift" \
|
||||
--ignore-filename-regex=".proto" \
|
||||
--ignore-filename-regex=".grpc.swift" \
|
||||
$(BUILD_BIN_DIR)/containerPackageTests.xctest/Contents/MacOS/containerPackageTests > $(COV_REPORT_FILE) ; \
|
||||
echo Code coverage report generated: $(COV_REPORT_FILE) ; \
|
||||
exit $${exit_code} ; \
|
||||
}
|
||||
@echo Merging integration coverage profdata...
|
||||
@xcrun llvm-profdata merge -sparse $(COVERAGE_OUTPUT_DIR)/integration/*/*.profraw -o $(COVERAGE_OUTPUT_DIR)/integration/default.profdata
|
||||
$(call GENERATE_COV_REPORTS,$(COVERAGE_OUTPUT_DIR)/integration/default.profdata,integration)
|
||||
|
||||
.PHONY: integration
|
||||
integration: init-block
|
||||
@@ -193,30 +277,9 @@ integration: init-block
|
||||
{ \
|
||||
exit_code=0; \
|
||||
CLITEST_LOG_ROOT=$(LOG_ROOT) && export CLITEST_LOG_ROOT ; \
|
||||
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIHelp || exit_code=1 ; \
|
||||
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIStatus || exit_code=1 ; \
|
||||
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIVersion || exit_code=1 ; \
|
||||
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLINetwork || exit_code=1 ; \
|
||||
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIRunLifecycle || exit_code=1 ; \
|
||||
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIRunCapabilities || exit_code=1 ; \
|
||||
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIExecCommand || exit_code=1 ; \
|
||||
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLICreateCommand || exit_code=1 ; \
|
||||
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIRunCommand1 || exit_code=1 ; \
|
||||
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIRunCommand2 || exit_code=1 ; \
|
||||
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIRunCommand3 || exit_code=1 ; \
|
||||
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIPruneCommand || exit_code=1 ; \
|
||||
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIRegistry || exit_code=1 ; \
|
||||
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIStatsCommand || exit_code=1 ; \
|
||||
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIImagesCommand || exit_code=1 ; \
|
||||
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIRunBase || exit_code=1 ; \
|
||||
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIRunInitImage || exit_code=1 ; \
|
||||
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIBuildBase || exit_code=1 ; \
|
||||
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIExportCommand || exit_code=1 ; \
|
||||
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIVolumes || exit_code=1 ; \
|
||||
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIKernelSet || exit_code=1 ; \
|
||||
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIAnonymousVolumes || exit_code=1 ; \
|
||||
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLINotFound || exit_code=1 ; \
|
||||
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLINoParallelCases || exit_code=1 ; \
|
||||
$(foreach suite,$(INTEGRATION_TEST_SUITES), \
|
||||
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter $(suite) || exit_code=1 ; \
|
||||
) \
|
||||
echo Ensuring apiserver stopped after the CLI integration tests ; \
|
||||
scripts/ensure-container-stopped.sh ; \
|
||||
exit $${exit_code} ; \
|
||||
@@ -285,4 +348,5 @@ clean:
|
||||
@rm -rf bin/ libexec/
|
||||
@rm -rf _site _serve
|
||||
@rm -f $(COV_REPORT_FILE)
|
||||
@rm -rf $(COVERAGE_OUTPUT_DIR)
|
||||
@$(SWIFT) package clean
|
||||
|
||||
Reference in New Issue
Block a user