From 57f0b9392bbee1998e6c7f3f25db222fe1dcdd12 Mon Sep 17 00:00:00 2001 From: Kathryn Baldauf Date: Tue, 15 Sep 2026 09:36:17 -0700 Subject: [PATCH] K8s plugin: Support custom CNI manifest (#2254) Signed-off-by: Kathryn Baldauf --- Sources/ContainerK8s/Commands/K8sCreate.swift | 10 +++ .../Support/K8sHelper+Bootstrap.swift | 21 ++++-- Tests/K8sPluginTests/K8sCreateCNITests.swift | 64 +++++++++++++++++++ docs/command-reference.md | 8 ++- docs/kubernetes.md | 35 ++++++++++ 5 files changed, 130 insertions(+), 8 deletions(-) create mode 100644 Tests/K8sPluginTests/K8sCreateCNITests.swift diff --git a/Sources/ContainerK8s/Commands/K8sCreate.swift b/Sources/ContainerK8s/Commands/K8sCreate.swift index 3f5d4415..3432240e 100644 --- a/Sources/ContainerK8s/Commands/K8sCreate.swift +++ b/Sources/ContainerK8s/Commands/K8sCreate.swift @@ -51,6 +51,9 @@ public struct K8sCreate: AsyncParsableCommand { @Option(help: "Node image reference (default: \(K8sHelper.nodeImage))") var nodeImage: String = K8sHelper.nodeImage + @Option(name: .long, help: "Optional path to a CNI manifest to apply.") + var cni: String? + public func run() async throws { LoggingSystem.bootstrap { _ in StderrLogHandler() } let log = Logger(label: K8sHelper.pluginName) @@ -59,6 +62,12 @@ public struct K8sCreate: AsyncParsableCommand { throw ContainerizationError(.invalidArgument, message: "cluster name \(name) is not a valid container ID") } + if let cni { + guard FileManager.default.fileExists(atPath: cni) else { + throw ContainerizationError(.invalidArgument, message: "CNI manifest not found at \(cni)") + } + } + let isTTY = isatty(FileHandle.standardError.fileDescriptor) == 1 let progressConfig = try ProgressConfig( showSpinner: isTTY, @@ -103,6 +112,7 @@ public struct K8sCreate: AsyncParsableCommand { try await K8sHelper.bootstrapControlPlane( nodeID: name, apiServerSANs: sans, advertiseAddress: vmIP, schedulable: provisioner.roles.contains(StandardRoles.worker), + cniManifestPath: cni, client: client, log: log) progress.set(description: "Waiting for cluster to be ready") diff --git a/Sources/ContainerK8s/Support/K8sHelper+Bootstrap.swift b/Sources/ContainerK8s/Support/K8sHelper+Bootstrap.swift index 964f9b72..58e6bec3 100644 --- a/Sources/ContainerK8s/Support/K8sHelper+Bootstrap.swift +++ b/Sources/ContainerK8s/Support/K8sHelper+Bootstrap.swift @@ -35,7 +35,7 @@ extension K8sHelper { static func bootstrapControlPlane( nodeID: String, apiServerSANs: [String], advertiseAddress: String, - schedulable: Bool, client: ContainerClient, log: Logger + schedulable: Bool, cniManifestPath: String? = nil, client: ContainerClient, log: Logger ) async throws { let configYAML = initConfigYAML(advertiseAddress: advertiseAddress, certSANs: apiServerSANs) var r = try await execCapture( @@ -73,11 +73,9 @@ extension K8sHelper { arguments: ["taint", "nodes", "--all", "node-role.kubernetes.io/control-plane-"]) } - log.info("Applying kindnet CNI", metadata: ["node": "\(nodeID)"]) - let manifest = try await loadKindnetManifest(log: log) - let apply = - "cat > /tmp/kindnet.yaml <<'EOF'\n\(manifest)\nEOF\n" - + "\(kubeconfigEnv) kubectl apply -f /tmp/kindnet.yaml" + log.info("Applying CNI manifest", metadata: ["node": "\(nodeID)"]) + let manifest = try await loadCNIManifest(path: cniManifestPath, log: log) + let apply = "\(kubeconfigEnv) kubectl apply -f - <<'EOF'\n\(manifest)\nEOF" r = try await execCapture( containerId: nodeID, executable: "/bin/sh", arguments: ["-c", apply], client: client) @@ -103,6 +101,17 @@ extension K8sHelper { return (token: parts[tokenIdx + 1], caCertHash: parts[hashIdx + 1]) } + static func loadCNIManifest(path: String?, log: Logger) async throws -> String { + if let path { + do { + return try String(contentsOfFile: path, encoding: .utf8) + } catch { + throw ContainerizationError(.invalidArgument, message: "failed to read CNI manifest at \(path): \(error)") + } + } + return try await loadKindnetManifest(log: log) + } + private static func loadKindnetManifest(log: Logger) async throws -> String { let pluginLoader = try await Utility.createPluginLoader(log: log) guard let plugin = pluginLoader.findPlugin(forExecutable: CommandLine.executablePath), diff --git a/Tests/K8sPluginTests/K8sCreateCNITests.swift b/Tests/K8sPluginTests/K8sCreateCNITests.swift new file mode 100644 index 00000000..cafde567 --- /dev/null +++ b/Tests/K8sPluginTests/K8sCreateCNITests.swift @@ -0,0 +1,64 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerizationError +import Foundation +import Logging +import Testing + +@testable import ContainerK8s + +// MARK: - K8sCreate flag parsing + +@Suite("K8sCreate --cni flag") +struct K8sCreateCNIFlagTests { + @Test func cniDefaultsToNilWhenNotProvided() throws { + let command = try K8sCreate.parse([]) + #expect(command.cni == nil) + } + + @Test func cniCapturesProvidedPath() throws { + let command = try K8sCreate.parse(["--cni", "/tmp/my-cni.yaml"]) + #expect(command.cni == "/tmp/my-cni.yaml") + } +} + +// MARK: - K8sHelper.loadCNIManifest + +@Suite("K8sHelper.loadCNIManifest") +struct LoadCNIManifestTests { + private let log = Logger(label: "test") + + @Test func customPathReturnsItsContents() async throws { + let contents = "kind: DaemonSet\nmetadata:\n name: my-custom-cni\n" + let dir = FileManager.default.temporaryDirectory + let url = dir.appendingPathComponent(UUID().uuidString + ".yaml") + try contents.write(to: url, atomically: true, encoding: .utf8) + defer { try? FileManager.default.removeItem(at: url) } + + let result = try await K8sHelper.loadCNIManifest(path: url.path, log: log) + #expect(result == contents) + } + + @Test func missingPathThrowsInvalidArgument() async throws { + let missingPath = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString + "-does-not-exist.yaml").path + + await #expect(throws: ContainerizationError.self) { + _ = try await K8sHelper.loadCNIManifest(path: missingPath, log: log) + } + } +} diff --git a/docs/command-reference.md b/docs/command-reference.md index a99ac852..9361036a 100644 --- a/docs/command-reference.md +++ b/docs/command-reference.md @@ -1605,18 +1605,19 @@ container system property list --format json ### `container k8s create` -Creates and starts a local Kubernetes cluster. Pulls the node image if needed, runs `kubeadm init`, installs the kindnet CNI, and merges the cluster credentials into `~/.kube/config`. +Creates and starts a local Kubernetes cluster. Pulls the node image if needed, runs `kubeadm init`, installs a CNI (default: bundled kindnet), and merges the cluster credentials into `~/.kube/config`. **Usage** ```bash -container k8s create [--name ] [--node-image ] [--rm] [] [--debug] +container k8s create [--name ] [--node-image ] [--cni ] [--rm] [] [--debug] ``` **Options** * `--name `: Cluster name (default: `k8s-dev`) * `--node-image `: Node image reference (default: `docker.io/kindest/node:v1.35.5`) +* `--cni `: Optional path to a CNI manifest to apply. If not provided, the bundled kindnet CNI is used. * `--rm`: Remove the cluster container after it stops **Resource Options** @@ -1643,6 +1644,9 @@ container k8s create --name my-cluster --cpus 4 --memory 8g # create a cluster that removes itself when stopped container k8s create --name temp-cluster --rm + +# create a cluster using a custom CNI manifest instead of the bundled kindnet +container k8s create --cni ./my-cni.yaml ``` ### `container k8s start` diff --git a/docs/kubernetes.md b/docs/kubernetes.md index a3656e2f..9e3e73a8 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -150,6 +150,41 @@ By default, clusters use `kindest/node:v1.35.5`, a Kubernetes-in-Docker image op container k8s create --node-image docker.io/kindest/node:v1.34.11 ``` +## Custom CNI + +By default, clusters install the bundled kindnet CNI for pod networking. Use `--cni` to apply a different CNI manifest instead: + +```bash +container k8s create --name my-cluster --cni ./my-cni.yaml +``` + +The manifest must be a plain Kubernetes YAML file (the same shape `kubectl apply -f` expects), not a Helm chart. + +### Example: Cilium + +Cilium is distributed as a Helm chart, so render a plain manifest from it first: + +```bash +helm repo add cilium https://helm.cilium.io/ +helm repo update +helm template cilium cilium/cilium --version 1.20.1 --namespace kube-system --set ipam.mode=kubernetes > cilium.yaml +``` + +`--set ipam.mode=kubernetes` avoids a CIDR conflict: the chart's default (`cluster-pool`, `10.0.0.0/8`) overlaps kubeadm's pod subnet and service CIDR on these clusters. + +Then create the cluster with that manifest: + +```bash +container k8s create --name cilium-demo --cni ./cilium.yaml +``` + +Verify Cilium came up: + +```bash +kubectl --context cilium-demo get pods -n kube-system +kubectl --context cilium-demo get nodes -o wide +``` + ## Cluster cleanup Remove a cluster and its data: