K8s plugin: Support custom CNI manifest (#2254)

Signed-off-by: Kathryn Baldauf <k_baldauf@apple.com>

Sources/ContainerK8s/Commands/K8sCreate.swift

@@ -51,6 +51,9 @@ public struct K8sCreate: AsyncParsableCommand {
     @Option(help: "Node image reference (default: \(K8sHelper.nodeImage))")
     var nodeImage: String = K8sHelper.nodeImage
 
+    @Option(name: .long, help: "Optional path to a CNI manifest to apply.")
+    var cni: String?
+
     public func run() async throws {
         LoggingSystem.bootstrap { _ in StderrLogHandler() }
         let log = Logger(label: K8sHelper.pluginName)
@@ -59,6 +62,12 @@ public struct K8sCreate: AsyncParsableCommand {
             throw ContainerizationError(.invalidArgument, message: "cluster name \(name) is not a valid container ID")
         }
 
+        if let cni {
+            guard FileManager.default.fileExists(atPath: cni) else {
+                throw ContainerizationError(.invalidArgument, message: "CNI manifest not found at \(cni)")
+            }
+        }
+
         let isTTY = isatty(FileHandle.standardError.fileDescriptor) == 1
         let progressConfig = try ProgressConfig(
             showSpinner: isTTY,
@@ -103,6 +112,7 @@ public struct K8sCreate: AsyncParsableCommand {
             try await K8sHelper.bootstrapControlPlane(
                 nodeID: name, apiServerSANs: sans, advertiseAddress: vmIP,
                 schedulable: provisioner.roles.contains(StandardRoles.worker),
+                cniManifestPath: cni,
                 client: client, log: log)
 
             progress.set(description: "Waiting for cluster to be ready")

Sources/ContainerK8s/Support/K8sHelper+Bootstrap.swift

@@ -35,7 +35,7 @@ extension K8sHelper {
 
     static func bootstrapControlPlane(
         nodeID: String, apiServerSANs: [String], advertiseAddress: String,
-        schedulable: Bool, client: ContainerClient, log: Logger
+        schedulable: Bool, cniManifestPath: String? = nil, client: ContainerClient, log: Logger
     ) async throws {
         let configYAML = initConfigYAML(advertiseAddress: advertiseAddress, certSANs: apiServerSANs)
         var r = try await execCapture(
@@ -73,11 +73,9 @@ extension K8sHelper {
                 arguments: ["taint", "nodes", "--all", "node-role.kubernetes.io/control-plane-"])
         }
 
-        log.info("Applying kindnet CNI", metadata: ["node": "\(nodeID)"])
-        let manifest = try await loadKindnetManifest(log: log)
-        let apply =
-            "cat > /tmp/kindnet.yaml <<'EOF'\n\(manifest)\nEOF\n"
-            + "\(kubeconfigEnv) kubectl apply -f /tmp/kindnet.yaml"
+        log.info("Applying CNI manifest", metadata: ["node": "\(nodeID)"])
+        let manifest = try await loadCNIManifest(path: cniManifestPath, log: log)
+        let apply = "\(kubeconfigEnv) kubectl apply -f - <<'EOF'\n\(manifest)\nEOF"
         r = try await execCapture(
             containerId: nodeID, executable: "/bin/sh",
             arguments: ["-c", apply], client: client)
@@ -103,6 +101,17 @@ extension K8sHelper {
         return (token: parts[tokenIdx + 1], caCertHash: parts[hashIdx + 1])
     }
 
+    static func loadCNIManifest(path: String?, log: Logger) async throws -> String {
+        if let path {
+            do {
+                return try String(contentsOfFile: path, encoding: .utf8)
+            } catch {
+                throw ContainerizationError(.invalidArgument, message: "failed to read CNI manifest at \(path): \(error)")
+            }
+        }
+        return try await loadKindnetManifest(log: log)
+    }
+
     private static func loadKindnetManifest(log: Logger) async throws -> String {
         let pluginLoader = try await Utility.createPluginLoader(log: log)
         guard let plugin = pluginLoader.findPlugin(forExecutable: CommandLine.executablePath),

Tests/K8sPluginTests/K8sCreateCNITests.swift

@@ -0,0 +1,64 @@
+//===----------------------------------------------------------------------===//
+// Copyright © 2026 Apple Inc. and the container project authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//===----------------------------------------------------------------------===//
+
+import ContainerizationError
+import Foundation
+import Logging
+import Testing
+
+@testable import ContainerK8s
+
+// MARK: - K8sCreate flag parsing
+
+@Suite("K8sCreate --cni flag")
+struct K8sCreateCNIFlagTests {
+    @Test func cniDefaultsToNilWhenNotProvided() throws {
+        let command = try K8sCreate.parse([])
+        #expect(command.cni == nil)
+    }
+
+    @Test func cniCapturesProvidedPath() throws {
+        let command = try K8sCreate.parse(["--cni", "/tmp/my-cni.yaml"])
+        #expect(command.cni == "/tmp/my-cni.yaml")
+    }
+}
+
+// MARK: - K8sHelper.loadCNIManifest
+
+@Suite("K8sHelper.loadCNIManifest")
+struct LoadCNIManifestTests {
+    private let log = Logger(label: "test")
+
+    @Test func customPathReturnsItsContents() async throws {
+        let contents = "kind: DaemonSet\nmetadata:\n  name: my-custom-cni\n"
+        let dir = FileManager.default.temporaryDirectory
+        let url = dir.appendingPathComponent(UUID().uuidString + ".yaml")
+        try contents.write(to: url, atomically: true, encoding: .utf8)
+        defer { try? FileManager.default.removeItem(at: url) }
+
+        let result = try await K8sHelper.loadCNIManifest(path: url.path, log: log)
+        #expect(result == contents)
+    }
+
+    @Test func missingPathThrowsInvalidArgument() async throws {
+        let missingPath = FileManager.default.temporaryDirectory
+            .appendingPathComponent(UUID().uuidString + "-does-not-exist.yaml").path
+
+        await #expect(throws: ContainerizationError.self) {
+            _ = try await K8sHelper.loadCNIManifest(path: missingPath, log: log)
+        }
+    }
+}

docs/command-reference.md

@@ -1605,18 +1605,19 @@ container system property list --format json
 
 ### `container k8s create`
 
-Creates and starts a local Kubernetes cluster. Pulls the node image if needed, runs `kubeadm init`, installs the kindnet CNI, and merges the cluster credentials into `~/.kube/config`.
+Creates and starts a local Kubernetes cluster. Pulls the node image if needed, runs `kubeadm init`, installs a CNI (default: bundled kindnet), and merges the cluster credentials into `~/.kube/config`.
 
 **Usage**
 
 ```bash
-container k8s create [--name <name>] [--node-image <image>] [--rm] [<resource options>] [--debug]
+container k8s create [--name <name>] [--node-image <image>] [--cni <path>] [--rm] [<resource options>] [--debug]
 ```
 
 **Options**
 
 *   `--name <name>`: Cluster name (default: `k8s-dev`)
 *   `--node-image <image>`: Node image reference (default: `docker.io/kindest/node:v1.35.5`)
+*   `--cni <path>`: Optional path to a CNI manifest to apply. If not provided, the bundled kindnet CNI is used.
 *   `--rm`: Remove the cluster container after it stops
 
 **Resource Options**
@@ -1643,6 +1644,9 @@ container k8s create --name my-cluster --cpus 4 --memory 8g
 
 # create a cluster that removes itself when stopped
 container k8s create --name temp-cluster --rm
+
+# create a cluster using a custom CNI manifest instead of the bundled kindnet
+container k8s create --cni ./my-cni.yaml
 ```
 
 ### `container k8s start`

docs/kubernetes.md

@@ -150,6 +150,41 @@ By default, clusters use `kindest/node:v1.35.5`, a Kubernetes-in-Docker image op
 container k8s create --node-image docker.io/kindest/node:v1.34.11
 ```
 
+## Custom CNI
+
+By default, clusters install the bundled kindnet CNI for pod networking. Use `--cni` to apply a different CNI manifest instead:
+
+```bash
+container k8s create --name my-cluster --cni ./my-cni.yaml
+```
+
+The manifest must be a plain Kubernetes YAML file (the same shape `kubectl apply -f` expects), not a Helm chart.
+
+### Example: Cilium
+
+Cilium is distributed as a Helm chart, so render a plain manifest from it first:
+
+```bash
+helm repo add cilium https://helm.cilium.io/
+helm repo update
+helm template cilium cilium/cilium --version 1.20.1 --namespace kube-system --set ipam.mode=kubernetes > cilium.yaml
+```
+
+`--set ipam.mode=kubernetes` avoids a CIDR conflict: the chart's default (`cluster-pool`, `10.0.0.0/8`) overlaps kubeadm's pod subnet and service CIDR on these clusters.
+
+Then create the cluster with that manifest:
+
+```bash
+container k8s create --name cilium-demo --cni ./cilium.yaml
+```
+
+Verify Cilium came up:
+
+```bash
+kubectl --context cilium-demo get pods -n kube-system
+kubectl --context cilium-demo get nodes -o wide
+```
+
 ## Cluster cleanup
 
 Remove a cluster and its data: