Integration coverage xpc helpers (#1551)

- This fixes the LLVM coverage data not properly being emitted for XPC
  services. It requires piping the `LLVM_PROFILE_FILE` environment
  variable through to all the services and plugins. The variable itself
  also required the "%c" formatter to ensure that it continuously emits
  coverage data, otherwise when XPC services are killed via "bootout" they
  do not emit coverage.

Makefile

@@ -16,11 +16,17 @@
 BUILD_CONFIGURATION ?= debug
 WARNINGS_AS_ERRORS ?= true
 SWIFT_CONFIGURATION := $(if $(filter-out false,$(WARNINGS_AS_ERRORS)),-Xswiftc -warnings-as-errors)
+# Code-coverage instrumentation, layered onto the shared build stages. Empty for
+# ordinary builds; the coverage-* targets opt in via a target-specific value so
+# only those goals compile instrumented binaries.
+COVERAGE_FLAG ?=
 export RELEASE_VERSION ?= $(shell git describe --tags --always)
 export GIT_COMMIT := $(shell git rev-parse HEAD)
 
 # Commonly used locations
 SWIFT := "/usr/bin/swift"
+# Shared swift build invocation; callers append --build-tests / --product / etc.
+SWIFT_BUILD = $(SWIFT) build -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION)
 DEST_DIR ?= /usr/local/
 ROOT_DIR := $(shell git rev-parse --show-toplevel)
 BUILD_BIN_DIR = $(shell $(SWIFT) build -c $(BUILD_CONFIGURATION) --show-bin-path)
@@ -56,13 +62,29 @@ all: init-block
 build:
 	@echo Building container binaries...
 	@$(SWIFT) --version
-	@$(SWIFT) build -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION)
+	@$(SWIFT_BUILD)
+
+.PHONY: build-tests
+# Shared build stage for every test target: builds the test bundle (and the
+# product binaries) once so the test targets can run with --skip-build. This is
+# a distinct target from `build` so `make all test` builds products and tests as
+# two separate steps rather than colliding on a single once-built target.
+# COVERAGE_FLAG instruments the binaries when set by the coverage-* targets.
+build-tests:
+	@echo Building container binaries and tests...
+	@$(SWIFT) --version
+	@$(SWIFT_BUILD) --build-tests $(COVERAGE_FLAG)
+
+.PHONY: coverage-all
+coverage-all: build-tests
+	@"$(MAKE)" BUILD_CONFIGURATION=$(BUILD_CONFIGURATION) DEST_DIR="$(ROOT_DIR)/" SUDO= install
+	@"$(MAKE)" init-block
 
 .PHONY: cli
 cli:
 	@echo Building container CLI...
 	@$(SWIFT) --version
-	@$(SWIFT) build -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --product container
+	@$(SWIFT_BUILD) --product container
 	@echo Installing container CLI to bin/...
 	@mkdir -p bin
 	@install "$(BUILD_BIN_DIR)/container" "bin/container"
@@ -150,8 +172,8 @@ dsym:
 	@(cd "$(dir $(DSYM_DIR))" ; zip -r $(notdir $(DSYM_PATH)) $(notdir $(DSYM_DIR)))
 
 .PHONY: test
-test:
-	@$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --skip TestCLI --skip IntegrationTests
+test: build-tests
+	@$(SWIFT) test --skip-build -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --skip TestCLI --skip IntegrationTests
 
 .PHONY: install-kernel
 install-kernel:
@@ -167,6 +189,16 @@ COV_DATA_DIR = $(shell $(SWIFT) test --show-coverage-path | xargs dirname)
 COV_REPORT_FILE = $(ROOT_DIR)/code-coverage-report
 COVERAGE_OUTPUT_DIR := $(ROOT_DIR)/coverage-reports
 TEST_BINARY = $(BUILD_BIN_DIR)/containerPackageTests.xctest/Contents/MacOS/containerPackageTests
+# All product binaries that may be instrumented for coverage.
+# Used as additional -object args to llvm-cov for integration/combined reports.
+COV_BINARIES := \
+	$(BUILD_BIN_DIR)/container \
+	$(BUILD_BIN_DIR)/container-apiserver \
+	$(BUILD_BIN_DIR)/container-runtime-linux \
+	$(BUILD_BIN_DIR)/container-network-vmnet \
+	$(BUILD_BIN_DIR)/container-core-images \
+	$(BUILD_BIN_DIR)/machine-apiserver
+COV_OBJECT_FLAGS := $(patsubst %,-object %,$(COV_BINARIES))
 # Set of files we do not want to get caught in the coverage generation
 LLVM_COV_IGNORE := \
 	--ignore-filename-regex=".build/" \
@@ -177,22 +209,27 @@ LLVM_COV_IGNORE := \
 	--ignore-filename-regex=".grpc.swift"
 
 # Generate JSON + HTML coverage reports and a coverage-percent.txt from a profdata file.
-# $(1) = profdata path, $(2) = tier name (unit/integration/combined)
+# $(1) = profdata path, $(2) = tier name (unit/integration/combined), $(3) = additional -object flags (optional)
 define GENERATE_COV_REPORTS
 	@echo Exporting $(2) coverage JSON...
 	@xcrun llvm-cov export --compilation-dir=`pwd` \
 		-instr-profile=$(1) \
 		$(LLVM_COV_IGNORE) \
-		$(TEST_BINARY) > $(COVERAGE_OUTPUT_DIR)/$(2)/coverage-summary.json
+		$(TEST_BINARY) $(3) > $(COVERAGE_OUTPUT_DIR)/$(2)/coverage-summary.json
 	@echo Generating $(2) coverage HTML report...
 	@xcrun llvm-cov show --compilation-dir=`pwd` --format=html \
 		-instr-profile=$(1) \
 		$(LLVM_COV_IGNORE) \
 		-output-dir=$(COVERAGE_OUTPUT_DIR)/$(2)/html \
-		$(TEST_BINARY)
+		$(TEST_BINARY) $(3)
 	@echo Extracting $(2) coverage percentages...
-	@jq -r '"line coverage: \(.data[0].totals.lines.percent | . * 100 | round | . / 100)%\nfunction coverage: \(.data[0].totals.functions.percent | . * 100 | round | . / 100)%"' \
+	@jq -r '.data[0].totals as $$t | \
+		"Coverage summary:", \
+		"  lines:     \($$t.lines.percent | . * 100 | round | . / 100)% (\($$t.lines.covered) of \($$t.lines.count))", \
+		"  functions: \($$t.functions.percent | . * 100 | round | . / 100)% (\($$t.functions.covered) of \($$t.functions.count))", \
+		"  regions:   \($$t.regions.percent | . * 100 | round | . / 100)% (\($$t.regions.covered) of \($$t.regions.count))"' \
 		$(COVERAGE_OUTPUT_DIR)/$(2)/coverage-summary.json > $(COVERAGE_OUTPUT_DIR)/$(2)/coverage-percent.txt
+	@echo "-- $(2) coverage --"
 	@cat $(COVERAGE_OUTPUT_DIR)/$(2)/coverage-percent.txt
 endef
 
@@ -215,6 +252,10 @@ SERIAL_FILTER = $(subst $(space),|,$(strip $(SERIAL_TEST_SUITES)))
 
 INTEGRATION_SWIFT_EXTRA ?=
 INTEGRATION_POST_TEST ?=
+# Environment prefix applied to the `container system start` invocation. Empty for
+# ordinary runs; coverage runs set LLVM_PROFILE_FILE here so launchd-managed helper
+# (XPC service) processes emit their own profraw data.
+INTEGRATION_PROFILE_ENV ?=
 
 PRESERVE_KERNELS ?= false
 # Default scratch root under the project directory so container build can access context
@@ -236,7 +277,7 @@ define RUN_INTEGRATION
 		fi ; \
 	fi
 	@echo Running the integration tests...
-	@bin/container --debug system start --timeout 60 --enable-kernel-install $(SYSTEM_START_OPTS) && \
+	@$(INTEGRATION_PROFILE_ENV) bin/container --debug system start --timeout 60 --enable-kernel-install $(SYSTEM_START_OPTS) && \
 	{ \
 		CLITEST_LOG_ROOT=$(LOG_ROOT) && export CLITEST_LOG_ROOT ; \
 		CLITEST_SCRATCH_ROOT=$(SCRATCH_ROOT) && export CLITEST_SCRATCH_ROOT ; \
@@ -261,34 +302,42 @@ integration: init-block
 
 .PHONY: coverage-integration
 coverage-integration: INTEGRATION_SWIFT_EXTRA = --skip-build --enable-code-coverage
-coverage-integration: INTEGRATION_POST_TEST = cp $(COV_DATA_DIR)/*.profraw $(COVERAGE_OUTPUT_DIR)/integration/ ;
-coverage-integration: all
+coverage-integration: INTEGRATION_POST_TEST = cp $(COV_DATA_DIR)/*.profraw $(COVERAGE_OUTPUT_DIR)/integration/ || true ;
+# Continuous mode (%c) mmaps the profraw and syncs counters live. The XPC helper
+# services are torn down by `launchctl bootout` (SIGTERM/SIGKILL) rather than
+# exiting cleanly, so a non-continuous profile (written by an atexit handler that
+# never runs on SIGKILL) would lose the helpers' counters. %p-%m keeps each
+# process/module profile in its own file so they don't collide.
+coverage-integration: INTEGRATION_PROFILE_ENV = LLVM_PROFILE_FILE=$(COVERAGE_OUTPUT_DIR)/integration/%p-%m%c.profraw
+coverage-integration: coverage-all
 	@mkdir -p $(COVERAGE_OUTPUT_DIR)/integration
+	@rm -f $(COVERAGE_OUTPUT_DIR)/integration/*.profraw
 	$(RUN_INTEGRATION)
+	@echo Merging integration coverage profdata...
+	@xcrun llvm-profdata merge -sparse $(COVERAGE_OUTPUT_DIR)/integration/*.profraw -o $(COVERAGE_OUTPUT_DIR)/integration/default.profdata
+	$(call GENERATE_COV_REPORTS,$(COVERAGE_OUTPUT_DIR)/integration/default.profdata,integration,$(COV_OBJECT_FLAGS))
 
 empty :=
 space := $(empty) $(empty)
 
+# Opt the coverage targets in to instrumentation. The value propagates to the
+# shared build-tests target so compilation is instrumented when necessary.
+coverage coverage-all coverage-unit coverage-integration: COVERAGE_FLAG = --enable-code-coverage -Xswiftc -DCONTAINER_COVERAGE
+
 .PHONY: coverage
-coverage: coverage-build coverage-unit coverage-integration
-	@echo Merging integration coverage profdata...
-	@xcrun llvm-profdata merge -sparse $(COVERAGE_OUTPUT_DIR)/integration/*.profraw -o $(COVERAGE_OUTPUT_DIR)/integration/default.profdata
-	$(call GENERATE_COV_REPORTS,$(COVERAGE_OUTPUT_DIR)/integration/default.profdata,integration)
+# Merge the per-tier profdata from coverage-unit and coverage-integration into a
+# combined report. Each prerequisite target produces its own tier report first.
+coverage: coverage-unit coverage-integration
 	@echo Merging combined coverage profdata...
 	@mkdir -p $(COVERAGE_OUTPUT_DIR)/combined
 	@xcrun llvm-profdata merge -sparse \
 		$(COVERAGE_OUTPUT_DIR)/unit/default.profdata \
 		$(COVERAGE_OUTPUT_DIR)/integration/default.profdata \
 		-o $(COVERAGE_OUTPUT_DIR)/combined/default.profdata
-	$(call GENERATE_COV_REPORTS,$(COVERAGE_OUTPUT_DIR)/combined/default.profdata,combined)
-
-.PHONY: coverage-build
-coverage-build:
-	@echo Building tests with coverage instrumentation...
-	@$(SWIFT) build --build-tests --enable-code-coverage -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION)
+	$(call GENERATE_COV_REPORTS,$(COVERAGE_OUTPUT_DIR)/combined/default.profdata,combined,$(COV_OBJECT_FLAGS))
 
 .PHONY: coverage-unit
-coverage-unit:
+coverage-unit: build-tests
 	@echo Running unit test coverage...
 	@rm -f $(COV_DATA_DIR)/*.profraw
 	@mkdir -p $(COVERAGE_OUTPUT_DIR)/unit
@@ -297,7 +346,6 @@ coverage-unit:
 	@xcrun llvm-profdata merge -sparse $(COV_DATA_DIR)/*.profraw -o $(COVERAGE_OUTPUT_DIR)/unit/default.profdata
 	$(call GENERATE_COV_REPORTS,$(COVERAGE_OUTPUT_DIR)/unit/default.profdata,unit)
 
-
 .PHONY: fmt
 fmt: swift-fmt update-licenses
 

Sources/ContainerPlugin/PluginLoader.swift

@@ -201,11 +201,19 @@ extension PluginLoader {
 }
 
 extension PluginLoader {
-    public static let proxyKeys = Set([
-        "http_proxy", "HTTP_PROXY",
-        "https_proxy", "HTTPS_PROXY",
-        "no_proxy", "NO_PROXY",
-    ])
+    public static let proxyKeys: Set<String> = {
+        var keys: Set<String> = [
+            "http_proxy", "HTTP_PROXY",
+            "https_proxy", "HTTPS_PROXY",
+            "no_proxy", "NO_PROXY",
+        ]
+        #if CONTAINER_COVERAGE
+        // Allows LLVM coverage profiling data to be written by launchd-managed
+        // helper processes. Compiled in only for coverage enabled builds.
+        keys.insert("LLVM_PROFILE_FILE")
+        #endif
+        return keys
+    }()
 
     public func registerWithLaunchd(
         plugin: Plugin,

Tests/ContainerPluginTests/PluginLoaderTest.swift

@@ -157,6 +157,19 @@ struct PluginLoaderTest {
             ])
     }
 
+    #if CONTAINER_COVERAGE
+    @Test
+    func testFilterEnvironmentWithLLVMProfileFile() async throws {
+        let env = [
+            "LLVM_PROFILE_FILE": "/tmp/coverage/%p-%m%c.profraw",
+            "OTHER_VAR": "value",
+        ]
+        let filtered = PluginLoader.filterEnvironment(env: env)
+
+        #expect(filtered == ["LLVM_PROFILE_FILE": "/tmp/coverage/%p-%m%c.profraw"])
+    }
+    #endif
+
     @Test
     func testFilterEnvironmentEmpty() async throws {
         let filtered = PluginLoader.filterEnvironment(env: [:])