K8s plugin (#2044)

- closes #2043 
- introduces the k8s plugin, allowing users to make single
  node clusters with the kind base image 
- other functionality is included as well such as creation,
  deletion, and loading custom images
This commit is contained in:
jshi991
2026-08-05 19:25:27 -07:00
committed by GitHub
parent 60612eef51
commit 0d111be4c6
21 changed files with 3176 additions and 1 deletions
+1 -1
View File
@@ -23,7 +23,7 @@ jobs:
buildAndTest:
name: Build and test the project
if: github.repository == 'apple/container'
timeout-minutes: 75
timeout-minutes: 90
runs-on: [self-hosted, macos, tahoe, ARM64]
permissions:
contents: read
+4
View File
@@ -133,6 +133,7 @@ $(STAGING_DIR):
@mkdir -p "$(join $(STAGING_DIR), libexec/container/plugins/container-core-images/bin)"
@mkdir -p "$(join $(STAGING_DIR), libexec/container/plugins/machine-apiserver/bin)"
@mkdir -p "$(join $(STAGING_DIR), libexec/container/plugins/machine-apiserver/resources)"
@mkdir -p "$(join $(STAGING_DIR), libexec/container/plugins/k8s/bin)"
@install "$(BUILD_BIN_DIR)/container" "$(join $(STAGING_DIR), bin/container)"
@install "$(BUILD_BIN_DIR)/container-apiserver" "$(join $(STAGING_DIR), bin/container-apiserver)"
@@ -146,6 +147,8 @@ $(STAGING_DIR):
@install Sources/Plugins/MachineAPIServer/config.toml "$(join $(STAGING_DIR), libexec/container/plugins/machine-apiserver/config.toml)"
@install Sources/Plugins/MachineAPIServer/Resources/init "$(join $(STAGING_DIR), libexec/container/plugins/machine-apiserver/resources/init)"
@install Sources/Plugins/MachineAPIServer/Resources/create-user.sh "$(join $(STAGING_DIR), libexec/container/plugins/machine-apiserver/resources/create-user.sh)"
@install "$(BUILD_BIN_DIR)/k8s" "$(join $(STAGING_DIR), libexec/container/plugins/k8s/bin/k8s)"
@install Sources/Plugins/K8s/config.toml "$(join $(STAGING_DIR), libexec/container/plugins/k8s/config.toml)"
@echo Install update script
@install scripts/update-container.sh "$(join $(STAGING_DIR), bin/update-container.sh)"
@@ -161,6 +164,7 @@ installer-pkg: $(STAGING_DIR)
@codesign $(CODESIGN_OPTS) --prefix=com.apple.container. --entitlements=signing/container-runtime-linux.entitlements "$(join $(STAGING_DIR), libexec/container/plugins/container-runtime-linux/bin/container-runtime-linux)"
@codesign $(CODESIGN_OPTS) --prefix=com.apple.container. --entitlements=signing/container-network-vmnet.entitlements "$(join $(STAGING_DIR), libexec/container/plugins/container-network-vmnet/bin/container-network-vmnet)"
@codesign $(CODESIGN_OPTS) --prefix=com.apple.container. "$(join $(STAGING_DIR), libexec/container/plugins/machine-apiserver/bin/machine-apiserver)"
@codesign $(CODESIGN_OPTS) --prefix=com.apple.container. "$(join $(STAGING_DIR), libexec/container/plugins/k8s/bin/k8s)"
@echo Creating application installer
@pkgbuild --root "$(STAGING_DIR)" --identifier com.apple.container-installer --install-location /usr/local --version ${RELEASE_VERSION} $(PKG_PATH)
+28
View File
@@ -164,6 +164,34 @@ let package = Package(
"ContainerResource",
]
),
.testTarget(
name: "K8sTests",
dependencies: [
"k8s",
"ContainerResource",
"Yams",
],
path: "Tests/K8sPluginTests"
),
.executableTarget(
name: "k8s",
dependencies: [
.product(name: "ArgumentParser", package: "swift-argument-parser"),
.product(name: "Logging", package: "swift-log"),
.product(name: "Containerization", package: "containerization"),
.product(name: "ContainerizationOCI", package: "containerization"),
"ContainerAPIClient",
"ContainerLog",
"ContainerPersistence",
"ContainerResource",
"ContainerVersion",
"TerminalProgress",
"Yams",
],
path: "Sources/Plugins/K8s",
exclude: ["config.toml"],
resources: [.process("Resources/kindnet.yaml")]
),
.executableTarget(
name: "container-apiserver",
dependencies: [
+59
View File
@@ -0,0 +1,59 @@
//===----------------------------------------------------------------------===//
// Copyright © 2026 Apple Inc. and the container project authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//===----------------------------------------------------------------------===//
import ArgumentParser
import ContainerVersion
@main
struct K8sCommand: AsyncParsableCommand {
static let configuration = CommandConfiguration(
commandName: "k8s",
abstract: "Manage local Kubernetes development clusters (EXPERIMENTAL)",
discussion: """
EXAMPLES:
Create a cluster by name and list clusters:
$ container k8s create --name my-cluster
$ container k8s list
Switch between clusters:
$ container k8s create --name second-cluster
$ kubectl config use-context second-cluster
$ kubectl config use-context my-cluster
Write the cluster context to an alternate configuration file:
$ container k8s write-config --name my-cluster --kubeconfig ~/.kube/my-cluster.kubeconfig
$ KUBECONFIG=~/.kube/my-cluster.kubeconfig kubectl cluster-info
Load a local image into the cluster and run it:
$ container image pull docker.io/library/hello-world:latest
$ container image tag docker.io/library/hello-world:latest my-hello-world:latest
$ container k8s load-image --name my-cluster my-hello-world:latest
$ kubectl run hello-job --image=my-hello-world:latest --restart=Never --attach --rm -i
Stop and delete the cluster:
$ container k8s delete --name my-cluster
""",
version: ReleaseVersion.singleLine(appName: "k8s"),
subcommands: [
K8sCreate.self,
K8sDelete.self,
K8sList.self,
K8sLoadImage.self,
K8sStart.self,
K8sWriteConfig.self,
]
)
}
+187
View File
@@ -0,0 +1,187 @@
//===----------------------------------------------------------------------===//
// Copyright © 2026 Apple Inc. and the container project authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//===----------------------------------------------------------------------===//
import ArgumentParser
import ContainerAPIClient
import ContainerLog
import ContainerPersistence
import ContainerResource
import ContainerizationError
import Darwin
import Foundation
import Logging
import TerminalProgress
struct K8sCreate: AsyncParsableCommand {
static let configuration = CommandConfiguration(
commandName: "create",
abstract: "Create and start a local Kubernetes cluster"
)
@Option(name: .long, help: "Cluster name (default: \(K8sHelper.defaultName))")
var name: String = K8sHelper.defaultName
@Flag(name: [.customLong("rm"), .long], help: "Remove the cluster container after it stops")
var remove: Bool = false
@OptionGroup(title: "Resource options")
var resourceFlags: Flags.Resource
@OptionGroup(title: "Registry options")
var registryFlags: Flags.Registry
@OptionGroup(title: "Image fetch options")
var imageFetchFlags: Flags.ImageFetch
@Option(help: "Node image reference (default: \(K8sHelper.nodeImage))")
var nodeImage: String = K8sHelper.nodeImage
func run() async throws {
LoggingSystem.bootstrap { _ in StderrLogHandler() }
let log = Logger(label: K8sHelper.pluginName)
guard ManagedContainer.nameValid(name) else {
throw ContainerizationError(.invalidArgument, message: "cluster name \(name) is not a valid container ID")
}
let isTTY = isatty(FileHandle.standardError.fileDescriptor) == 1
let progressConfig = try ProgressConfig(
showSpinner: isTTY,
showTasks: true,
showItems: true,
ignoreSmallSize: true,
totalTasks: 2, // fetch image, unpack image
clearOnFinish: isTTY,
outputMode: isTTY ? .ansi : .plain
)
let progress = ProgressBar(config: progressConfig)
defer { progress.finish() }
progress.start()
let containerSystemConfig: ContainerSystemConfig = try await ConfigurationLoader.load()
try await K8sHelper.ensureImage(nodeImage: nodeImage, log: log, containerSystemConfig: containerSystemConfig)
let fqdn = K8sHelper.fqdn(for: name, domain: containerSystemConfig.dns.domain)
let dns = Flags.DNS(domain: nil, nameservers: [], options: [], searchDomains: [])
let management = Flags.Management(
arch: Arch.hostArchitecture().rawValue,
capAdd: ["ALL"],
capDrop: [],
cidfile: "",
detach: true,
dns: dns,
dnsDisabled: false,
entrypoint: nil,
initImage: nil,
kernel: nil,
kernelArgs: [],
labels: [
"\(ResourceLabelKeys.plugin)=\(K8sHelper.pluginName)",
"\(ResourceLabelKeys.role)=\(K8sHelper.controlPlaneRoleName)",
],
maskedPaths: [],
mounts: [],
name: name,
networks: [],
os: "linux",
platform: nil,
publishPorts: fqdn == nil ? [try await K8sHelper.clusterPort()] : [],
publishSockets: [],
readOnly: false,
readonlyPaths: [],
remove: remove,
rosetta: true,
runtime: nil,
ssh: false,
shmSize: nil,
tmpFs: [],
useInit: false,
virtualization: false,
volumes: []
)
let updatedResource = K8sHelper.defaultedResourceFlags(resourceFlags)
let processFlags = Flags.Process(cwd: nil, env: K8sHelper.nodeProxyEnv(), envFile: [], gid: nil, interactive: false, tty: false, uid: nil, ulimits: [], user: nil)
var (config, kernel, initfs) = try await Utility.containerConfigFromFlags(
id: name,
image: nodeImage,
arguments: [],
process: processFlags,
management: management,
resource: updatedResource,
registry: registryFlags,
imageFetch: imageFetchFlags,
containerSystemConfig: containerSystemConfig,
progressUpdate: progress.handler,
log: log
)
// Allow the node to modify /proc/sys (e.g. net.ipv4.ip_forward) during setup.
config.maskedPaths = []
config.readonlyPaths = []
let client = ContainerClient()
let options = ContainerCreateOptions(autoRemove: remove)
try await client.create(
configuration: config,
options: options,
kernel: kernel,
initImage: initfs
)
progress.set(description: "Starting cluster")
let io = try ProcessIO.create(tty: false, interactive: false, detach: true)
defer { try? io.close() }
let process = try await client.bootstrap(id: name, stdio: io.stdio)
try await process.start()
try io.closeAfterStart()
progress.set(description: "Waiting for node to boot")
try await K8sHelper.waitForNodeBooted(containerId: name, client: client, log: log)
let snapshot = try await client.get(id: name)
guard let vmIP = snapshot.networks.first?.ipv4Address.address.description else {
throw ContainerizationError(.internalError, message: "no VM IP for control plane \(name)")
}
var sans = ["127.0.0.1"]
if let fqdn { sans.append(contentsOf: [vmIP, fqdn]) }
progress.set(description: "Running kubeadm init")
try await K8sHelper.prepareNode(nodeID: name, client: client, log: log)
try await K8sHelper.bootstrapControlPlane(
nodeID: name, apiServerSANs: sans, advertiseAddress: vmIP,
client: client, log: log)
progress.set(description: "Waiting for cluster to be ready")
try await K8sHelper.waitForReady(containerId: name, client: client, log: log)
progress.set(description: "Writing kubeconfig")
do {
let rawConfig = try await K8sHelper.fetchConfig(containerId: name, client: client, log: log)
let kubeConfig = try await K8sHelper.transformConfig(rawConfig, containerId: name, fqdn: fqdn, client: client)
try K8sHelper.mergeConfig(kubeConfig, containerId: name, setCurrentContext: true, log: log)
} catch {
log.warning("failed to write kubeconfig", metadata: ["name": "\(name)", "error": "\(error)"])
log.info("cluster is running; use 'container k8s write-config --name \(name)' to write the kubeconfig")
}
progress.finish()
print(name)
}
}
+57
View File
@@ -0,0 +1,57 @@
//===----------------------------------------------------------------------===//
// Copyright © 2026 Apple Inc. and the container project authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//===----------------------------------------------------------------------===//
import ArgumentParser
import ContainerAPIClient
import ContainerLog
import ContainerResource
import ContainerizationError
import Logging
struct K8sDelete: AsyncParsableCommand {
static let configuration = CommandConfiguration(
commandName: "delete",
abstract: "Delete a Kubernetes cluster",
aliases: ["rm"]
)
@Option(name: .long, help: "Cluster name (default: \(K8sHelper.defaultName))")
var name: String = K8sHelper.defaultName
func run() async throws {
LoggingSystem.bootstrap { _ in StderrLogHandler() }
let log = Logger(label: K8sHelper.pluginName)
let client = ContainerClient()
if let container = try? await client.get(id: name) {
guard container.configuration.labels[ResourceLabelKeys.plugin] == K8sHelper.pluginName else {
log.error("container is not a k8s cluster, refusing delete", metadata: ["name": "\(name)"])
throw ContainerizationError(.invalidArgument, message: "\(name) is not a k8s cluster")
}
}
do {
try? await client.stop(id: name)
try await client.delete(id: name)
} catch let error as ContainerizationError where error.code == .notFound {
log.debug("cluster container not found, skipping delete", metadata: ["name": "\(name)"])
}
try K8sHelper.removeConfig(containerId: name, log: log)
print(name)
}
}
+679
View File
@@ -0,0 +1,679 @@
//===----------------------------------------------------------------------===//
// Copyright © 2026 Apple Inc. and the container project authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//===----------------------------------------------------------------------===//
import ContainerAPIClient
import ContainerPersistence
import ContainerResource
import ContainerVersion
import ContainerizationError
import ContainerizationOCI
import ContainerizationOS
import Darwin
import Foundation
import Logging
import SystemPackage
import Yams
// MARK: - ListDisplayable
protocol ListDisplayable {
static var tableHeader: [String] { get }
var tableRow: [String] { get }
var quietValue: String { get }
}
// MARK: - TableOutput
struct TableOutput: Sendable {
private let rows: [[String]]
private let spacing: Int
init(rows: [[String]], spacing: Int = 2) {
self.rows = rows
self.spacing = spacing
}
func format() -> String {
var output = ""
let maxLengths = self.maxLength()
for rowIndex in 0..<self.rows.count {
let row = self.rows[rowIndex]
for columnIndex in 0..<row.count - 1 {
let currentLength = (maxLengths[columnIndex] ?? 0) + self.spacing
let padded = row[columnIndex].padding(toLength: currentLength, withPad: " ", startingAt: 0)
output += padded
}
output += row.last ?? ""
output += (rowIndex == self.rows.count - 1) ? "" : "\n"
}
return output
}
private func maxLength() -> [Int: Int] {
var output: [Int: Int] = [:]
for row in self.rows {
for (i, column) in row.enumerated() {
let currentMax = output[i] ?? 0
output[i] = (column.count > currentMax) ? column.count : currentMax
}
}
return output
}
}
// MARK: - K8sHelper
struct K8sHelper {
static let pluginName: String = "k8s"
static let defaultName: String = "k8s-dev"
static let controlPlaneRoleName: String = "control-plane"
private static var defaultCPUs: Int64 {
Int64(max(ProcessInfo.processInfo.processorCount / 4, 2))
}
private static var defaultMemory: String {
let gb = Int(ProcessInfo.processInfo.physicalMemory / (1024 * 1024 * 1024)) / 4
return "\(max(gb, 2))g"
}
static let nodeImage = "docker.io/kindest/node:v1.35.5@sha256:ce977ae6d65918d0b58a5f8b5e940429c2ce42fa3a5619ec2bbc60b949c0ac95"
private static let kubeconfigPath = "/etc/kubernetes/admin.conf"
private static let kubeconfigEnv = "KUBECONFIG=\(kubeconfigPath)"
private static let kubectlPath = "/bin/kubectl"
private static let kubeadmPath = "/usr/bin/kubeadm"
private static let ignorePreflightErrors =
"Swap,SystemVerification,FileContent--proc-sys-net-bridge-bridge-nf-call-iptables"
private static let podSubnet = "10.244.0.0/16"
// kubeadm default service subnet; must stay in sync if ClusterConfiguration.serviceSubnet is ever set.
private static let serviceSubnet = "10.96.0.0/12"
// Proxy env var names forwarded from the host into the cluster container.
static let proxyEnvVars = ["HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy"]
// Returns proxy env vars with NO_PROXY augmented to bypass internal cluster CIDRs.
// Without this, kubelet routes apiserver traffic through the host proxy and times out.
static func nodeProxyEnv() -> [String] {
let bypassCIDRs = "192.168.0.0/16,\(podSubnet),\(serviceSubnet)"
let hostEnv = ProcessInfo.processInfo.environment
return proxyEnvVars.map { name in
guard name.uppercased() == "NO_PROXY" else { return name }
let existing = hostEnv[name] ?? hostEnv[name == "NO_PROXY" ? "no_proxy" : "NO_PROXY"] ?? ""
let augmented = existing.isEmpty ? bypassCIDRs : "\(existing),\(bypassCIDRs)"
return "\(name)=\(augmented)"
}
}
private static let clusterContainerPort: UInt16 = 6443
private static let clusterHostPortBase: UInt16 = 6445
private static func findAvailableHostPort(excluding: Set<UInt16> = []) throws -> UInt16 {
var port = clusterHostPortBase
while port < UInt16.max {
if excluding.contains(port) {
port += 1
continue
}
let sock = Darwin.socket(AF_INET, SOCK_STREAM, 0)
guard sock >= 0 else {
throw ContainerizationError(.internalError, message: "socket() failed while probing for available port")
}
var addr = sockaddr_in()
addr.sin_family = sa_family_t(AF_INET)
addr.sin_port = port.bigEndian
addr.sin_addr.s_addr = INADDR_ANY
let available = withUnsafePointer(to: &addr) { ptr in
ptr.withMemoryRebound(to: sockaddr.self, capacity: 1) {
Darwin.bind(sock, $0, socklen_t(MemoryLayout<sockaddr_in>.size)) == 0
}
}
Darwin.close(sock)
if available { return port }
port += 1
}
throw ContainerizationError(.internalError, message: "no available host port found above \(clusterHostPortBase)")
}
static func clusterPort() async throws -> String {
let snapshots = try await ContainerClient().list(
filters: ContainerListFilters(labels: [ResourceLabelKeys.plugin: pluginName])
)
let reserved = Set(snapshots.flatMap { $0.configuration.publishedPorts.map(\.hostPort) })
let port = try findAvailableHostPort(excluding: reserved)
return "\(port):\(clusterContainerPort)"
}
private static let kubeconfigDir: FilePath = FilePath(
FileManager.default.homeDirectoryForCurrentUser.path(percentEncoded: false)
).appending(".kube")
// MARK: - Resource defaults
static func defaultedResourceFlags(_ flags: Flags.Resource) -> Flags.Resource {
var f = flags
if f.cpus == nil { f.cpus = defaultCPUs }
if f.memory == nil { f.memory = defaultMemory }
return f
}
// MARK: - Image management
static func ensureImage(nodeImage: String = K8sHelper.nodeImage, log: Logger, containerSystemConfig: ContainerSystemConfig) async throws {
do {
_ = try await ClientImage.get(reference: nodeImage, containerSystemConfig: containerSystemConfig)
log.debug("k8s node image present", metadata: ["ref": "\(nodeImage)"])
return
} catch let error as ContainerizationError where error.code == .notFound {
log.info("Pulling k8s node image", metadata: ["ref": "\(nodeImage)"])
}
let platform = try Platform(from: "linux/\(Arch.hostArchitecture().rawValue)")
_ = try await ClientImage.fetch(
reference: nodeImage,
platform: platform,
scheme: .auto,
containerSystemConfig: containerSystemConfig,
progressUpdate: nil)
}
// MARK: - Node bootstrap
private static func execCapture(
containerId: String, executable: String, arguments: [String],
client: ContainerClient
) async throws -> (code: Int32, output: String) {
let pipe = Pipe()
let config = ProcessConfiguration(
executable: executable, arguments: arguments, environment: [], terminal: false)
let proc = try await client.createProcess(
containerId: containerId, processId: UUID().uuidString.lowercased(),
configuration: config, stdio: [nil, pipe.fileHandleForWriting, nil])
try await proc.start()
pipe.fileHandleForWriting.closeFile()
let data = pipe.fileHandleForReading.readDataToEndOfFile()
try? pipe.fileHandleForReading.close()
let code = try await proc.wait()
return (code, String(data: data, encoding: .utf8) ?? "")
}
static func prepareNode(nodeID: String, client: ContainerClient, log: Logger) async throws {
log.info("Preparing node", metadata: ["id": "\(nodeID)"])
let result = try await execCapture(
containerId: nodeID, executable: "/bin/sh",
arguments: ["-c", nodePrepScript], client: client)
guard result.code == 0 else {
throw ContainerizationError(.internalError, message: "node prep failed on \(nodeID): \(result.output)")
}
}
static func bootstrapControlPlane(
nodeID: String, apiServerSANs: [String], advertiseAddress: String,
client: ContainerClient, log: Logger
) async throws {
let configYAML = initConfigYAML(advertiseAddress: advertiseAddress, certSANs: apiServerSANs)
var r = try await execCapture(
containerId: nodeID, executable: "/bin/sh",
arguments: ["-c", "cat > /etc/kubernetes/kubeadm-config.yaml <<'EOF'\n\(configYAML)\nEOF"],
client: client)
guard r.code == 0 else {
throw ContainerizationError(.internalError, message: "write kubeadm config failed on \(nodeID): \(r.output)")
}
log.info("Running kubeadm init", metadata: ["node": "\(nodeID)"])
r = try await execCapture(
containerId: nodeID, executable: kubeadmPath,
arguments: [
"init", "--config", "/etc/kubernetes/kubeadm-config.yaml",
"--ignore-preflight-errors", ignorePreflightErrors,
],
client: client)
guard r.code == 0 else {
throw ContainerizationError(.internalError, message: "kubeadm init failed on \(nodeID): \(r.output)")
}
r = try await execCapture(
containerId: nodeID, executable: "/bin/sh",
arguments: ["-c", "mkdir -p /root/.kube && cp \(kubeconfigPath) /root/.kube/config"],
client: client)
guard r.code == 0 else {
throw ContainerizationError(.internalError, message: "failed to install root kubeconfig on \(nodeID): \(r.output)")
}
log.info("Removing control-plane taint for single-node scheduling", metadata: ["node": "\(nodeID)"])
_ = try await runProbe(
client: client, containerId: nodeID,
arguments: ["taint", "nodes", "--all", "node-role.kubernetes.io/control-plane-"])
log.info("Applying kindnet CNI", metadata: ["node": "\(nodeID)"])
let manifest = try loadKindnetManifest()
let apply =
"cat > /tmp/kindnet.yaml <<'EOF'\n\(manifest)\nEOF\n"
+ "\(kubeconfigEnv) kubectl apply -f /tmp/kindnet.yaml"
r = try await execCapture(
containerId: nodeID, executable: "/bin/sh",
arguments: ["-c", apply], client: client)
guard r.code == 0 else {
throw ContainerizationError(.internalError, message: "apply CNI failed on \(nodeID): \(r.output)")
}
}
private static func loadKindnetManifest() throws -> String {
guard let url = Bundle.module.url(forResource: "kindnet", withExtension: "yaml"),
let contents = try? String(contentsOf: url, encoding: .utf8)
else {
throw ContainerizationError(.internalError, message: "kindnet manifest resource missing")
}
return contents
}
private static let nodePrepScript: String = {
"""
set -e
mkdir -p /etc/containerd/conf.d
cat > /etc/containerd/conf.d/native-snapshotter.toml <<'EOF'
[plugins.'io.containerd.cri.v1.images']
snapshotter = "native"
EOF
sysctl -w net.ipv4.ip_forward=1 2>/dev/null || true
sysctl -w net.bridge.bridge-nf-call-iptables=1 2>/dev/null || true
sysctl -w net.bridge.bridge-nf-call-ip6tables=1 2>/dev/null || true
systemctl restart containerd
ctr -n k8s.io images tag registry.k8s.io/pause:3.10 registry.k8s.io/pause:3.10.1 2>/dev/null || true
iptables -t mangle -A OUTPUT -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1220
iptables -t mangle -A FORWARD -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1220
"""
}()
private static func initConfigYAML(advertiseAddress: String, certSANs: [String]) -> String {
let sans = certSANs.map { " - \($0)" }.joined(separator: "\n")
return """
apiVersion: kubeadm.k8s.io/v1beta4
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: \(advertiseAddress)
bindPort: 6443
nodeRegistration:
criSocket: unix:///run/containerd/containerd.sock
---
apiVersion: kubeadm.k8s.io/v1beta4
kind: ClusterConfiguration
kubernetesVersion: \(kubernetesVersion())
networking:
podSubnet: \(podSubnet)
apiServer:
certSANs:
\(sans)
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cgroupDriver: systemd
failSwapOn: false
"""
}
private static func kubernetesVersion() -> String {
let nameAndTag = nodeImage.split(separator: "@").first.map(String.init) ?? nodeImage
guard let ref = try? Reference.parse(nameAndTag), let tag = ref.tag else { return "v1.35" }
return tag
}
// MARK: - FQDN detection
static func fqdn(for name: String, domain: String?) -> String? {
if name.contains(".") { return name }
guard let domain, !domain.isEmpty else { return nil }
return "\(name).\(domain)"
}
static func detectFQDN(name: String) async -> String? {
let domain = try? await ConfigurationLoader.load().dns.domain
return fqdn(for: name, domain: domain)
}
// MARK: - Readiness
static func waitForNodeBooted(containerId: String, client: ContainerClient, log: Logger) async throws {
let timeout = 120
log.info("Waiting for node to boot", metadata: ["node": "\(containerId)"])
for attempt in 1...timeout {
let result = try await execCapture(
containerId: containerId, executable: "/bin/sh",
arguments: ["-c", "test -S /run/containerd/containerd.sock"], client: client)
if result.code == 0 { return }
if attempt == timeout {
log.info("check container logs with 'container logs \(containerId)'")
throw ContainerizationError(
.timeout,
message: "node \(containerId) did not boot within \(timeout * 2)s: containerd socket not present at /run/containerd/containerd.sock"
)
}
try await Task.sleep(for: .seconds(2))
}
}
private static func runProbe(client: ContainerClient, containerId: String, arguments: [String]) async throws -> Int32 {
let devNull = FileHandle(forWritingAtPath: "/dev/null")
defer { try? devNull?.close() }
let probe = ProcessConfiguration(
executable: kubectlPath,
arguments: arguments,
environment: [kubeconfigEnv],
terminal: false)
let proc = try await client.createProcess(
containerId: containerId,
processId: UUID().uuidString.lowercased(),
configuration: probe,
stdio: [nil, devNull, devNull])
try await proc.start()
return try await proc.wait()
}
static func waitForReady(containerId: String, client: ContainerClient, log: Logger) async throws {
let nodeReadyTimeout = 180
let podReadyTimeout = 300
log.info("Waiting for control-plane node to become ready")
for attempt in 1...nodeReadyTimeout {
let code: Int32
do {
code = try await runProbe(
client: client, containerId: containerId,
arguments: ["wait", "--for=condition=Ready", "node", "--all", "--timeout=2s"])
} catch {
throw ContainerizationError(
.internalError, message: "k8s cluster \(containerId) stopped unexpectedly during startup: \(error)")
}
if code == 0 { break }
if attempt == nodeReadyTimeout {
log.info("inspect node state with 'container exec \(containerId) kubectl get nodes -o wide'")
throw ContainerizationError(
.timeout,
message: "k8s cluster \(containerId) control-plane node did not become Ready within \(nodeReadyTimeout * 2)s"
)
}
try await Task.sleep(for: .seconds(2))
}
log.info("Waiting for kube-system pods to become ready")
for attempt in 1...podReadyTimeout {
let code: Int32
do {
code = try await runProbe(
client: client, containerId: containerId,
arguments: ["wait", "--for=condition=Available", "deployment/coredns", "-n", "kube-system", "--timeout=2s"])
} catch {
throw ContainerizationError(
.internalError, message: "k8s cluster \(containerId) stopped unexpectedly during startup: \(error)")
}
if code == 0 { return }
if attempt < podReadyTimeout {
try await Task.sleep(for: .seconds(2))
}
}
log.info("inspect pod state with 'container exec \(containerId) kubectl get pods -n kube-system'")
throw ContainerizationError(
.timeout,
message: "k8s cluster \(containerId) kube-system pods did not become available within \(podReadyTimeout * 2)s"
)
}
// MARK: - Kubeconfig
static func fetchConfig(containerId: String, client: ContainerClient, log: Logger) async throws -> KubeConfig {
log.info("Fetching kubeconfig", metadata: ["cluster": "\(containerId)"])
let container = try await client.get(id: containerId)
guard container.configuration.labels[ResourceLabelKeys.plugin] == pluginName else {
log.error("container is not a k8s cluster, refusing config fetch", metadata: ["name": "\(containerId)"])
throw ContainerizationError(.invalidArgument, message: "\(containerId) is not a k8s cluster")
}
let (exitCode, yaml) = try await execCapture(
containerId: containerId, executable: "/bin/cat",
arguments: [kubeconfigPath], client: client)
guard exitCode == 0 else {
throw ContainerizationError(.internalError, message: "failed to read kubeconfig from \(containerId): exit \(exitCode)")
}
do {
return try YAMLDecoder().decode(KubeConfig.self, from: yaml)
} catch {
throw ContainerizationError(.internalError, message: "failed to decode kubeconfig from \(containerId): \(error)")
}
}
static func transformConfig(_ config: KubeConfig, containerId: String, fqdn: String?, client: ContainerClient) async throws -> KubeConfig {
var config = config
let serverAddress: String
if let fqdn {
serverAddress = "https://\(fqdn):6443"
} else {
let snapshot = try await client.get(id: containerId)
guard
let hostPort = snapshot.configuration.publishedPorts
.first(where: { $0.containerPort == clusterContainerPort })?.hostPort
else {
throw ContainerizationError(.internalError, message: "no published port for cluster \(containerId)")
}
serverAddress = "https://127.0.0.1:\(hostPort)"
}
for i in config.clusters.indices {
config.clusters[i].cluster.server = serverAddress
}
// Rename all entries to containerId. kubeadm uses fixed names ("kubernetes",
// "kubernetes-admin@kubernetes", etc.) rather than "default", so we rename
// unconditionally and fix up the cross-references in the context.
config.clusters = config.clusters.map {
var c = $0
c.name = containerId
return c
}
config.users = config.users.map {
var u = $0
u.name = containerId
return u
}
config.contexts = config.contexts.map {
var nc = $0
nc.name = containerId
nc.context.cluster = containerId
nc.context.user = containerId
return nc
}
config.currentContext = containerId
return config
}
static func resolveKubeconfigMergePath() -> FilePath {
let defaultPath = kubeconfigDir.appending("config")
guard let raw = Darwin.getenv("KUBECONFIG") else { return defaultPath }
let env = String(cString: raw)
guard !env.isEmpty else { return defaultPath }
let paths = env.split(separator: ":").map(String.init).filter { !$0.isEmpty }
guard !paths.isEmpty else { return defaultPath }
if paths.count == 1 { return FilePath(paths[0]) }
for p in paths where FileManager.default.fileExists(atPath: p) {
return FilePath(p)
}
return FilePath(paths[paths.count - 1])
}
static func mergeConfig(_ config: KubeConfig, containerId: String, targetPath: FilePath? = nil, setCurrentContext: Bool = false, log: Logger) throws {
let path = targetPath ?? resolveKubeconfigMergePath()
log.info("Writing kubeconfig", metadata: ["cluster": "\(containerId)", "path": "\(path)"])
let targetDir = path.removingLastComponent()
try FileManager.default.createDirectory(atPath: targetDir.string, withIntermediateDirectories: true)
var existing: KubeConfig
if FileManager.default.fileExists(atPath: path.string) {
do {
let yaml = try String(contentsOfFile: path.string, encoding: .utf8)
existing = try YAMLDecoder().decode(KubeConfig.self, from: yaml)
} catch {
throw ContainerizationError(
.internalError,
message: "kubeconfig at \(path) could not be parsed: \(error)"
)
}
} else {
existing = .empty
}
existing.clusters.removeAll { $0.name == containerId }
existing.contexts.removeAll { $0.name == containerId }
existing.users.removeAll { $0.name == containerId }
existing.clusters.append(contentsOf: config.clusters)
existing.contexts.append(contentsOf: config.contexts)
existing.users.append(contentsOf: config.users)
if setCurrentContext && existing.currentContext == nil {
existing.currentContext = containerId
}
let output = try YAMLEncoder().encode(existing)
try output.write(toFile: path.string, atomically: true, encoding: .utf8)
}
static func removeConfig(containerId: String, log: Logger) throws {
let path = resolveKubeconfigMergePath()
log.info("Removing kubeconfig", metadata: ["cluster": "\(containerId)", "path": "\(path)"])
guard FileManager.default.fileExists(atPath: path.string) else { return }
let existing: KubeConfig
do {
let yaml = try String(contentsOfFile: path.string, encoding: .utf8)
existing = try YAMLDecoder().decode(KubeConfig.self, from: yaml)
} catch {
log.warning("kubeconfig exists but could not be parsed, skipping removal", metadata: ["path": "\(path)", "error": "\(error)"])
return
}
var updated = existing
updated.clusters.removeAll { $0.name == containerId }
updated.contexts.removeAll { $0.name == containerId }
updated.users.removeAll { $0.name == containerId }
if updated.currentContext == containerId {
updated.currentContext = nil
}
let output = try YAMLEncoder().encode(updated)
try output.write(toFile: path.string, atomically: true, encoding: .utf8)
}
// MARK: - List rows
static func buildK8sRows(from snapshots: [ContainerSnapshot]) -> [K8sNodeResource] {
var controlPlanes: [ContainerSnapshot] = []
var workers: [ContainerSnapshot] = []
for snapshot in snapshots {
switch snapshot.configuration.labels[ResourceLabelKeys.role] {
case controlPlaneRoleName: controlPlanes.append(snapshot)
default: workers.append(snapshot)
}
}
var rows: [K8sNodeResource] = []
var assignedWorkerIDs = Set<String>()
for cp in controlPlanes.sorted(by: { $0.configuration.id < $1.configuration.id }) {
let clusterName = cp.configuration.id
rows.append(K8sNodeResource(clusterName: clusterName, snapshot: cp))
let cpWorkers =
workers
.filter { $0.configuration.id.hasPrefix("\(clusterName)-worker-") }
.sorted { $0.configuration.id < $1.configuration.id }
for w in cpWorkers {
rows.append(K8sNodeResource(clusterName: clusterName, snapshot: w))
assignedWorkerIDs.insert(w.configuration.id)
}
}
for w
in workers
.filter({ !assignedWorkerIDs.contains($0.configuration.id) })
.sorted(by: { $0.configuration.id < $1.configuration.id })
{
let clusterName = w.configuration.id
.components(separatedBy: "-worker-").dropLast().joined(separator: "-worker-")
rows.append(K8sNodeResource(clusterName: clusterName, snapshot: w))
}
return rows
}
static func renderTable<T: ListDisplayable>(_ items: [T]) -> String {
var rows: [[String]] = [T.tableHeader]
for item in items {
rows.append(item.tableRow)
}
return TableOutput(rows: rows).format()
}
// MARK: - Image reference helpers
static func isShortName(_ reference: String) -> Bool {
guard let ref = try? Reference.parse(reference) else { return true }
return ref.domain == nil
}
static func fqReference(_ reference: String) -> String {
guard let ref = try? Reference.parse(reference) else { return reference }
ref.normalize()
return ref.description
}
}
// MARK: - K8sNodeResource
struct K8sNodeResource: ManagedResource, ListDisplayable {
let clusterName: String
let snapshot: ContainerSnapshot
// MARK: ManagedResource
var id: String { snapshot.configuration.id }
var name: String { snapshot.configuration.id }
var creationDate: Date { snapshot.configuration.creationDate }
var labels: ResourceLabels { (try? ResourceLabels(snapshot.configuration.labels)) ?? .init() }
static func nameValid(_ name: String) -> Bool { ManagedContainer.nameValid(name) }
static var tableHeader: [String] {
["CLUSTER", "NODE", "ROLE", "STATE", "CPUS", "MEMORY", "ADDR", "PORTS"]
}
var tableRow: [String] {
let role = snapshot.configuration.labels[ResourceLabelKeys.role] ?? ""
let addr = snapshot.networks.map { $0.ipv4Address.address.description }.joined(separator: ",")
let memoryMB = snapshot.configuration.resources.memoryInBytes / (1024 * 1024)
let ports = snapshot.configuration.publishedPorts
.map { "\($0.hostPort)->\($0.containerPort)" }
.joined(separator: ",")
return [
clusterName,
snapshot.configuration.id,
role,
snapshot.status.rawValue,
"\(snapshot.configuration.resources.cpus)",
"\(memoryMB) MB",
addr,
ports,
]
}
var quietValue: String { snapshot.configuration.id }
}
+39
View File
@@ -0,0 +1,39 @@
//===----------------------------------------------------------------------===//
// Copyright © 2026 Apple Inc. and the container project authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//===----------------------------------------------------------------------===//
import ArgumentParser
import ContainerAPIClient
import ContainerLog
import ContainerResource
import Logging
struct K8sList: AsyncParsableCommand {
static let configuration = CommandConfiguration(
commandName: "list",
abstract: "List clusters and their nodes",
aliases: ["ls"]
)
func run() async throws {
LoggingSystem.bootstrap { _ in StderrLogHandler() }
let snapshots = try await ContainerClient().list(
filters: ContainerListFilters(labels: [ResourceLabelKeys.plugin: K8sHelper.pluginName])
)
let rows = K8sHelper.buildK8sRows(from: snapshots)
print(K8sHelper.renderTable(rows))
}
}
+122
View File
@@ -0,0 +1,122 @@
//===----------------------------------------------------------------------===//
// Copyright © 2026 Apple Inc. and the container project authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//===----------------------------------------------------------------------===//
import ArgumentParser
import ContainerAPIClient
import ContainerLog
import ContainerPersistence
import ContainerResource
import ContainerizationError
import ContainerizationOCI
import Foundation
import Logging
import SystemPackage
struct K8sLoadImage: AsyncParsableCommand {
private static let ctrPath = "/usr/local/bin/ctr"
static let configuration = CommandConfiguration(
commandName: "load-image",
abstract: "Load a container image into a cluster's containerd"
)
@Option(name: .long, help: "Cluster name (default: \(K8sHelper.defaultName))")
var name: String = K8sHelper.defaultName
@Argument(help: "Image reference to load (e.g. demo-api:latest)")
var image: String
@Option(
help: "Platform of the image to load (format: os/arch[/variant], default: linux/arm64)"
)
var platform: String?
func run() async throws {
LoggingSystem.bootstrap { _ in StderrLogHandler() }
let log = Logger(label: K8sHelper.pluginName)
let tmpFile = FilePath(FileManager.default.temporaryDirectory.path(percentEncoded: false))
.appending("k8s-image-\(UUID().uuidString).tar")
defer { try? FileManager.default.removeItem(atPath: tmpFile.string) }
let containerSystemConfig: ContainerSystemConfig = try await ConfigurationLoader.load()
let client = ContainerClient()
// Guard: refuse to operate on a container not owned by this plugin.
let container = try await client.get(id: name)
guard container.configuration.labels[ResourceLabelKeys.plugin] == K8sHelper.pluginName else {
log.error("container is not a k8s cluster, refusing image load", metadata: ["name": "\(name)"])
throw ContainerizationError(.invalidArgument, message: "\(name) is not a k8s cluster")
}
log.info("Saving image", metadata: ["ref": "\(image)"])
let fq = K8sHelper.fqReference(image)
let resolvedPlatform = try platform.map { try Platform(from: $0) } ?? Platform(from: "linux/\(Arch.hostArchitecture().rawValue)")
try await ClientImage.save(references: [fq], out: tmpFile.string, platform: resolvedPlatform, containerSystemConfig: containerSystemConfig)
log.info("Importing image into cluster", metadata: ["target": "\(name)"])
guard let inputHandle = FileHandle(forReadingAtPath: tmpFile.string) else {
throw ContainerizationError(.internalError, message: "failed to open image tar: \(tmpFile)")
}
defer { try? inputHandle.close() }
let importConfig = ProcessConfiguration(
executable: Self.ctrPath,
arguments: ["--namespace", "k8s.io", "images", "import", "-"],
environment: [],
terminal: false
)
let importProc = try await client.createProcess(
containerId: name,
processId: UUID().uuidString.lowercased(),
configuration: importConfig,
stdio: [inputHandle, nil, nil]
)
try await importProc.start()
let importCode = try await importProc.wait()
guard importCode == 0 else {
throw ContainerizationError(
.internalError,
message: "ctr import exited \(importCode) on \(name)")
}
// Tag with the fully-qualified docker.io/library/ name that kubelet expects,
// but only for short (unqualified) references.
if fq != image {
log.info("Tagging image for kubelet", metadata: ["short": "\(image)", "fq": "\(fq)"])
let tagConfig = ProcessConfiguration(
executable: Self.ctrPath,
arguments: ["--namespace", "k8s.io", "images", "tag", fq, image],
environment: [],
terminal: false
)
let tagProc = try await client.createProcess(
containerId: name,
processId: UUID().uuidString.lowercased(),
configuration: tagConfig,
stdio: [nil, nil, nil]
)
try await tagProc.start()
let tagCode = try await tagProc.wait()
guard tagCode == 0 else {
throw ContainerizationError(
.internalError,
message: "ctr tag exited \(tagCode) on \(name)")
}
}
}
}
+72
View File
@@ -0,0 +1,72 @@
//===----------------------------------------------------------------------===//
// Copyright © 2026 Apple Inc. and the container project authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//===----------------------------------------------------------------------===//
import ArgumentParser
import ContainerAPIClient
import ContainerLog
import ContainerResource
import ContainerizationError
import Foundation
import Logging
struct K8sStart: AsyncParsableCommand {
static let configuration = CommandConfiguration(
commandName: "start",
abstract: "Start a stopped Kubernetes cluster"
)
@Option(name: .long, help: "Cluster name (default: \(K8sHelper.defaultName))")
var name: String = K8sHelper.defaultName
func run() async throws {
LoggingSystem.bootstrap { _ in StderrLogHandler() }
let log = Logger(label: K8sHelper.pluginName)
let client = ContainerClient()
let container = try await client.get(id: name)
guard container.configuration.labels[ResourceLabelKeys.plugin] == K8sHelper.pluginName else {
log.error("container is not a k8s cluster, refusing start", metadata: ["name": "\(name)"])
throw ContainerizationError(.invalidArgument, message: "\(name) is not a k8s cluster")
}
if container.status == .running {
print(name)
return
}
let io = try ProcessIO.create(tty: false, interactive: false, detach: true)
defer { try? io.close() }
let process = try await client.bootstrap(id: name, stdio: io.stdio)
try await process.start()
try io.closeAfterStart()
try await K8sHelper.waitForNodeBooted(containerId: name, client: client, log: log)
try await K8sHelper.waitForReady(containerId: name, client: client, log: log)
do {
let fqdn = await K8sHelper.detectFQDN(name: name)
let rawConfig = try await K8sHelper.fetchConfig(containerId: name, client: client, log: log)
let config = try await K8sHelper.transformConfig(rawConfig, containerId: name, fqdn: fqdn, client: client)
try K8sHelper.mergeConfig(config, containerId: name, log: log)
} catch {
log.warning("failed to write kubeconfig", metadata: ["name": "\(name)", "error": "\(error)"])
log.info("cluster is running; use 'container k8s write-config --name \(name)' to write the kubeconfig")
}
print(name)
}
}
+47
View File
@@ -0,0 +1,47 @@
//===----------------------------------------------------------------------===//
// Copyright © 2026 Apple Inc. and the container project authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//===----------------------------------------------------------------------===//
import ArgumentParser
import ContainerAPIClient
import ContainerLog
import Foundation
import Logging
import SystemPackage
struct K8sWriteConfig: AsyncParsableCommand {
static let configuration = CommandConfiguration(
commandName: "write-config",
abstract: "Write the cluster context to a Kubernetes configuration file"
)
@Option(name: .long, help: "Cluster name (default: \(K8sHelper.defaultName))")
var name: String = K8sHelper.defaultName
@Option(name: .long, help: "Path to the kubeconfig file to write or append to (default: ~/.kube/config)")
var kubeconfig: String?
func run() async throws {
LoggingSystem.bootstrap { _ in StderrLogHandler() }
let log = Logger(label: K8sHelper.pluginName)
let targetPath = kubeconfig.map { FilePath($0) }
let client = ContainerClient()
let fqdn = await K8sHelper.detectFQDN(name: name)
let rawConfig = try await K8sHelper.fetchConfig(containerId: name, client: client, log: log)
let config = try await K8sHelper.transformConfig(rawConfig, containerId: name, fqdn: fqdn, client: client)
try K8sHelper.mergeConfig(config, containerId: name, targetPath: targetPath, log: log)
}
}
+168
View File
@@ -0,0 +1,168 @@
//===----------------------------------------------------------------------===//
// Copyright © 2026 Apple Inc. and the container project authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//===----------------------------------------------------------------------===//
import Foundation
// MARK: - KubeConfig
struct KubeConfig: Codable {
enum CodingKeys: String, CodingKey {
case apiVersion, kind, clusters, contexts, users
case currentContext = "current-context"
}
var apiVersion: String = "v1"
var kind: String = "Config"
var clusters: [NamedCluster] = []
var contexts: [NamedContext] = []
var users: [NamedAuthInfo] = []
var currentContext: String?
init() {}
init(from decoder: Decoder) throws {
let c = try decoder.container(keyedBy: CodingKeys.self)
apiVersion = try c.decodeIfPresent(String.self, forKey: .apiVersion) ?? "v1"
kind = try c.decodeIfPresent(String.self, forKey: .kind) ?? "Config"
clusters = try c.decodeIfPresent([NamedCluster].self, forKey: .clusters) ?? []
contexts = try c.decodeIfPresent([NamedContext].self, forKey: .contexts) ?? []
users = try c.decodeIfPresent([NamedAuthInfo].self, forKey: .users) ?? []
currentContext = try c.decodeIfPresent(String.self, forKey: .currentContext)
}
static let empty = KubeConfig()
}
// MARK: - NamedCluster
struct NamedCluster: Codable {
var name: String
var cluster: Cluster
}
// MARK: - Cluster
struct Cluster: Codable {
enum CodingKeys: String, CodingKey {
case server
case tlsServerName = "tls-server-name"
case insecureSkipTLSVerify = "insecure-skip-tls-verify"
case certificateAuthority = "certificate-authority"
case certificateAuthorityData = "certificate-authority-data"
case proxyURL = "proxy-url"
}
var server: String
var tlsServerName: String?
var insecureSkipTLSVerify: Bool?
var certificateAuthority: String?
var certificateAuthorityData: String?
var proxyURL: String?
}
// MARK: - NamedContext
struct NamedContext: Codable {
var name: String
var context: Context
}
// MARK: - Context
struct Context: Codable {
var cluster: String
var user: String
var namespace: String?
}
// MARK: - NamedAuthInfo
struct NamedAuthInfo: Codable {
enum CodingKeys: String, CodingKey {
case name
case authInfo = "user"
}
var name: String
var authInfo: AuthInfo
}
// MARK: - AuthInfo
struct AuthInfo: Codable {
enum CodingKeys: String, CodingKey {
case clientCertificate = "client-certificate"
case clientCertificateData = "client-certificate-data"
case clientKey = "client-key"
case clientKeyData = "client-key-data"
case token
case tokenFile = "token-file"
case impersonate
case impersonateGroups = "impersonate-groups"
case impersonateUserExtra = "impersonate-user-extra"
case username
case password
case authProvider = "auth-provider"
case exec
}
var clientCertificate: String?
var clientCertificateData: String?
var clientKey: String?
var clientKeyData: String?
var token: String?
var tokenFile: String?
var impersonate: String?
var impersonateGroups: [String]?
var impersonateUserExtra: [String: String]?
var username: String?
var password: String?
var authProvider: AuthProviderConfig?
var exec: ExecConfig?
}
// MARK: - AuthProviderConfig
struct AuthProviderConfig: Codable {
var name: String
var config: [String: String]?
}
// MARK: - ExecConfig
struct ExecConfig: Codable {
enum CodingKeys: String, CodingKey {
case command, args, env, apiVersion
case installHint = "installHint"
case provideClusterInfo = "provideClusterInfo"
case interactiveMode = "interactiveMode"
}
var command: String
var args: [String]?
var env: [ExecEnvVar]?
var apiVersion: String
var installHint: String?
var provideClusterInfo: Bool?
var interactiveMode: String?
}
// MARK: - ExecEnvVar
struct ExecEnvVar: Codable {
var name: String
var value: String
}
+135
View File
@@ -0,0 +1,135 @@
# kindnetd networking manifest
#
# Pinned copy of kind's embedded default CNI manifest, transcribed verbatim
# from kubernetes-sigs/kind pkg/build/nodeimage/const_cni.go at tag v0.29.0:
# https://github.com/kubernetes-sigs/kind/blob/v0.29.0/pkg/build/nodeimage/const_cni.go
#
# The POD_SUBNET value must stay in sync with podSubnet in K8sHelper.
#
# kindnetd image pinned by the same kind release: docker.io/kindest/kindnetd:v20260528-9350166c
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: kindnet
rules:
- apiGroups:
- policy
resources:
- podsecuritypolicies
verbs:
- use
resourceNames:
- kindnet
- apiGroups:
- ""
resources:
- nodes
- pods
- namespaces
verbs:
- list
- watch
- apiGroups:
- "networking.k8s.io"
resources:
- networkpolicies
verbs:
- list
- watch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: kindnet
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: kindnet
subjects:
- kind: ServiceAccount
name: kindnet
namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: kindnet
namespace: kube-system
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: kindnet
namespace: kube-system
labels:
tier: node
app: kindnet
k8s-app: kindnet
spec:
selector:
matchLabels:
app: kindnet
template:
metadata:
labels:
tier: node
app: kindnet
k8s-app: kindnet
spec:
hostNetwork: true
nodeSelector:
kubernetes.io/os: linux
tolerations:
- operator: Exists
serviceAccountName: kindnet
containers:
- name: kindnet-cni
image: docker.io/kindest/kindnetd:v20260528-9350166c
env:
- name: HOST_IP
valueFrom:
fieldRef:
fieldPath: status.hostIP
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: POD_SUBNET
value: "10.244.0.0/16"
volumeMounts:
- name: cni-cfg
mountPath: /etc/cni/net.d
- name: xtables-lock
mountPath: /run/xtables.lock
readOnly: false
- name: lib-modules
mountPath: /lib/modules
readOnly: true
- name: nri-plugin
mountPath: /var/run/nri
resources:
requests:
cpu: "100m"
memory: "50Mi"
limits:
cpu: "100m"
memory: "50Mi"
securityContext:
privileged: false
capabilities:
add: ["NET_RAW", "NET_ADMIN"]
volumes:
- name: cni-cfg
hostPath:
path: /etc/cni/net.d
- name: xtables-lock
hostPath:
path: /run/xtables.lock
type: FileOrCreate
- name: lib-modules
hostPath:
path: /lib/modules
- name: nri-plugin
hostPath:
path: /var/run/nri
+3
View File
@@ -0,0 +1,3 @@
abstract = "Local Kubernetes development cluster management"
author = "Apple"
version = 0.1
@@ -0,0 +1,101 @@
//===----------------------------------------------------------------------===//
// Copyright © 2026 Apple Inc. and the container project authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//===----------------------------------------------------------------------===//
import ContainerTestSupport
import Foundation
import Testing
@Suite(.serialized)
struct TestK8sLoadImageSerial {
private static let testImage = WarmupImage.alpine320.rawValue
private func dumpNodeDiagnostics(_ f: ContainerFixture, node: String) {
print("=== NODE DIAGNOSTICS [\(node)] ===")
let cmds: [(label: String, args: [String])] = [
("ip-link", ["ip", "link", "show"]),
("iptables-mss", ["iptables", "-t", "mangle", "-L", "-n", "-v"]),
("containerd", ["systemctl", "status", "containerd", "--no-pager", "-l"]),
("kubelet-log", ["journalctl", "-u", "kubelet", "--no-pager", "-n", "60"]),
("crictl-images", ["crictl", "images"]),
]
for (label, args) in cmds {
if let r = try? f.run(["exec", node] + args) {
let out = r.output.trimmingCharacters(in: .whitespacesAndNewlines)
let err = r.error.trimmingCharacters(in: .whitespacesAndNewlines)
print("[\(label)] exit=\(r.status)")
if !out.isEmpty { print(out) }
if !err.isEmpty { print("stderr: \(err)") }
}
}
print("=== END NODE DIAGNOSTICS ===")
}
private func imageExistsInNode(_ f: ContainerFixture, node: String, image: String) throws -> Bool {
print("[k8s-load] ctr images list (node: \(node), checking: \(image))")
let result = try f.run(["exec", node, "ctr", "--namespace", "k8s.io", "images", "list"])
print("[k8s-load] ctr images list exit=\(result.status) found=\(result.output.contains(image))")
guard result.status == 0 else { return false }
return result.output.contains(image)
}
private func dumpEnv(_ f: ContainerFixture, clusterName: String) {
print("=== ENV DUMP [\(clusterName)] ===")
if let result = try? f.run(["system", "status"]) {
print("[system status]\n\(result.output)")
}
if let result = try? f.run(["list"]) {
print("[container list]\n\(result.output)")
}
if let result = try? f.run(["image", "list"]) {
print("[image list]\n\(result.output)")
}
if let result = try? f.run(["inspect", clusterName]) {
print("[inspect \(clusterName)]\nstdout: \(result.output)\nstderr: \(result.error)")
}
print("=== END ENV DUMP ===")
}
@Test func testLoadImageIntoSingleNode() async throws {
try await ContainerFixture.with { f in
let name = "k8s-\(f.testID)"
f.addCleanup { _ = try? f.run(["k8s", "delete", "--name", name]) }
print("[k8s-load] k8s create --name \(name)")
let result = try f.run(["k8s", "create", "--name", name])
print("[k8s-load] k8s create exit=\(result.status)")
if result.status != 0 {
print("[k8s-load] k8s create stderr: \(result.error)")
dumpEnv(f, clusterName: name)
dumpNodeDiagnostics(f, node: name)
}
try result.check()
print("[k8s-load] pulling \(Self.testImage)")
try f.doPull(Self.testImage)
print("[k8s-load] k8s load-image --name \(name) \(Self.testImage)")
let loadResult = try f.run(["k8s", "load-image", "--name", name, Self.testImage])
print("[k8s-load] k8s load-image exit=\(loadResult.status)")
if loadResult.status != 0 {
print("[k8s-load] load-image stderr: \(loadResult.error)")
}
#expect(loadResult.status == 0)
#expect(try imageExistsInNode(f, node: name, image: "alpine"))
}
}
}
@@ -0,0 +1,213 @@
//===----------------------------------------------------------------------===//
// Copyright © 2026 Apple Inc. and the container project authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//===----------------------------------------------------------------------===//
import ContainerTestSupport
import Foundation
import Testing
@Suite(.serialized)
struct TestK8sNetworkingSerial {
private static let testImage = WarmupImage.alpine320.rawValue
private func dumpNodeDiagnostics(_ f: ContainerFixture, node: String) {
print("=== NODE DIAGNOSTICS [\(node)] ===")
let cmds: [(label: String, args: [String])] = [
("ip-link", ["ip", "link", "show"]),
("iptables-mss", ["iptables", "-t", "mangle", "-L", "-n", "-v"]),
("containerd", ["systemctl", "status", "containerd", "--no-pager", "-l"]),
("kubelet-log", ["journalctl", "-u", "kubelet", "--no-pager", "-n", "60"]),
("crictl-images", ["crictl", "images"]),
]
for (label, args) in cmds {
if let r = try? f.run(["exec", node] + args) {
let out = r.output.trimmingCharacters(in: .whitespacesAndNewlines)
let err = r.error.trimmingCharacters(in: .whitespacesAndNewlines)
print("[\(label)] exit=\(r.status)")
if !out.isEmpty { print(out) }
if !err.isEmpty { print("stderr: \(err)") }
}
}
print("=== END NODE DIAGNOSTICS ===")
}
private func dumpEnv(_ f: ContainerFixture, clusterName: String) {
print("=== ENV DUMP [\(clusterName)] ===")
if let result = try? f.run(["system", "status"]) {
print("[system status]\n\(result.output)")
}
if let result = try? f.run(["list"]) {
print("[container list]\n\(result.output)")
}
if let result = try? f.run(["inspect", clusterName]) {
print("[inspect \(clusterName)] stdout: \(result.output) stderr: \(result.error)")
}
print("=== END ENV DUMP ===")
}
@discardableResult
private func kubectl(_ f: ContainerFixture, node: String, args: [String]) throws -> (output: String, status: Int32) {
print("[k8s-net] kubectl \(args.joined(separator: " ")) (node: \(node))")
let result = try f.run(["exec", node, "kubectl"] + args)
print("[k8s-net] kubectl exit=\(result.status) output=\(result.output.prefix(120).trimmingCharacters(in: .whitespacesAndNewlines))")
let filteredStderr = result.error.components(separatedBy: "\n")
.filter { !$0.contains("Warning! Running debug build") && !$0.isEmpty }
.joined(separator: "\n")
if !filteredStderr.isEmpty {
print("[k8s-net] kubectl stderr: \(filteredStderr.prefix(300))")
}
return (result.output, result.status)
}
private func waitForPod(_ f: ContainerFixture, node: String, podName: String, timeout: Int = 300) throws {
print("[k8s-net] waitForPod \(podName) on \(node) (timeout=\(timeout)s)")
let (_, status) = try kubectl(
f, node: node,
args: [
"wait", "--for=condition=Ready", "pod/\(podName)", "--timeout=\(timeout)s",
])
guard status == 0 else {
let (podStatus, _) = try kubectl(f, node: node, args: ["get", "pod", podName, "--no-headers"])
print("[k8s-net] pod \(podName) status: \(podStatus.trimmingCharacters(in: .whitespacesAndNewlines))")
let (podDesc, _) = try kubectl(f, node: node, args: ["describe", "pod", podName])
print("[k8s-net] pod \(podName) describe:\n\(podDesc.prefix(1000))")
throw CommandError.executionFailed("pod \(podName) did not become ready within \(timeout)s")
}
print("[k8s-net] pod \(podName) is Ready")
}
// Verify that a pod schedules, reaches Running, and can be exec'd into.
@Test func testPodsScheduleAndRun() async throws {
try await ContainerFixture.with { f in
let name = "k8s-\(f.testID)"
f.addCleanup { _ = try? f.run(["k8s", "delete", "--name", name]) }
print("[k8s-net] k8s create --name \(name)")
let result = try f.run(["k8s", "create", "--name", name])
print("[k8s-net] k8s create exit=\(result.status)")
if result.status != 0 {
print("[k8s-net] k8s create stderr: \(result.error)")
dumpEnv(f, clusterName: name)
dumpNodeDiagnostics(f, node: name)
}
try result.check()
print("[k8s-net] pulling \(Self.testImage)")
try f.doPull(Self.testImage)
print("[k8s-net] k8s load-image --name \(name) \(Self.testImage)")
let loadResult = try f.run(["k8s", "load-image", "--name", name, Self.testImage])
print("[k8s-net] k8s load-image exit=\(loadResult.status)")
if loadResult.status != 0 { print("[k8s-net] k8s load-image stderr: \(loadResult.error)") }
#expect(loadResult.status == 0)
let (_, createStatus) = try kubectl(
f, node: name,
args: [
"run", "test-pod",
"--image=\(Self.testImage)",
"--image-pull-policy=Never",
"--restart=Never",
"--", "sleep", "300",
])
#expect(createStatus == 0)
try waitForPod(f, node: name, podName: "test-pod")
let (output, execStatus) = try kubectl(
f, node: name,
args: [
"exec", "test-pod", "--", "echo", "hello",
])
#expect(execStatus == 0)
#expect(output.contains("hello"))
}
}
// Verify pod-to-service communication and CoreDNS resolution on a single node.
@Test func testPodToServiceCommunication() async throws {
try await ContainerFixture.with { f in
let name = "k8s-\(f.testID)"
f.addCleanup { _ = try? f.run(["k8s", "delete", "--name", name]) }
print("[k8s-net] k8s create --name \(name)")
let result = try f.run(["k8s", "create", "--name", name])
print("[k8s-net] k8s create exit=\(result.status)")
if result.status != 0 {
print("[k8s-net] k8s create stderr: \(result.error)")
dumpEnv(f, clusterName: name)
}
try result.check()
print("[k8s-net] pulling \(Self.testImage)")
try f.doPull(Self.testImage)
print("[k8s-net] k8s load-image --name \(name) \(Self.testImage)")
let loadResult = try f.run(["k8s", "load-image", "--name", name, Self.testImage])
print("[k8s-net] k8s load-image exit=\(loadResult.status)")
if loadResult.status != 0 { print("[k8s-net] k8s load-image stderr: \(loadResult.error)") }
#expect(loadResult.status == 0)
// Server: alpine busybox httpd serving a static response.
let (_, serverStatus) = try kubectl(
f, node: name,
args: [
"run", "server",
"--image=\(Self.testImage)",
"--image-pull-policy=Never",
"--restart=Never",
"--port=8080",
"--", "sh", "-c",
"while true; do printf 'HTTP/1.1 200 OK\\r\\nContent-Length: 2\\r\\nConnection: close\\r\\n\\r\\nok' | nc -l -p 8080; done",
])
#expect(serverStatus == 0)
try waitForPod(f, node: name, podName: "server")
// Expose server as a ClusterIP service.
let (_, exposeStatus) = try kubectl(
f, node: name,
args: [
"expose", "pod", "server", "--port=8080", "--name=server-svc",
])
#expect(exposeStatus == 0)
// Client pod that stays alive so we can exec into it.
let (_, clientStatus) = try kubectl(
f, node: name,
args: [
"run", "client",
"--image=\(Self.testImage)",
"--image-pull-policy=Never",
"--restart=Never",
"--", "sleep", "300",
])
#expect(clientStatus == 0)
try waitForPod(f, node: name, podName: "client")
// Reach server via the service DNS name exercises CoreDNS + kube-proxy.
print("[k8s-net] wget from client to server-svc:8080")
let (response, wgetStatus) = try kubectl(
f, node: name,
args: [
"exec", "client", "--", "sh", "-c",
"sleep 2 && wget -qO- http://server-svc:8080",
])
print("[k8s-net] wget exit=\(wgetStatus) response=\(response.trimmingCharacters(in: .whitespacesAndNewlines))")
#expect(wgetStatus == 0)
#expect(response.contains("ok"))
}
}
}
@@ -0,0 +1,128 @@
//===----------------------------------------------------------------------===//
// Copyright © 2026 Apple Inc. and the container project authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//===----------------------------------------------------------------------===//
import ContainerTestSupport
import Foundation
import SystemPackage
import Testing
import Yams
@Suite(.serialized)
struct TestK8sRunSerial {
private func dumpNodeDiagnostics(_ f: ContainerFixture, node: String) {
print("=== NODE DIAGNOSTICS [\(node)] ===")
let cmds: [(label: String, args: [String])] = [
("ip-link", ["ip", "link", "show"]),
("iptables-mss", ["iptables", "-t", "mangle", "-L", "-n", "-v"]),
("containerd", ["systemctl", "status", "containerd", "--no-pager", "-l"]),
("kubelet-log", ["journalctl", "-u", "kubelet", "--no-pager", "-n", "60"]),
("crictl-images", ["crictl", "images"]),
]
for (label, args) in cmds {
if let r = try? f.run(["exec", node] + args) {
let out = r.output.trimmingCharacters(in: .whitespacesAndNewlines)
let err = r.error.trimmingCharacters(in: .whitespacesAndNewlines)
print("[\(label)] exit=\(r.status)")
if !out.isEmpty { print(out) }
if !err.isEmpty { print("stderr: \(err)") }
}
}
print("=== END NODE DIAGNOSTICS ===")
}
private func loadKubeconfig() throws -> [String: Any] {
let path = FilePath(FileManager.default.homeDirectoryForCurrentUser.path)
.appending(".kube")
.appending("config")
let yaml = try String(contentsOfFile: path.string, encoding: .utf8)
guard let parsed = try Yams.load(yaml: yaml) as? [String: Any] else {
throw CommandError.executionFailed("could not parse kubeconfig at \(path.string)")
}
return parsed
}
@Test func testRunSingleNode() async throws {
try await ContainerFixture.with { f in
let name = "k8s-\(f.testID)"
f.addCleanup { _ = try? f.run(["k8s", "delete", "--name", name]) }
print("[k8s-run] k8s create --name \(name)")
let result = try f.run(["k8s", "create", "--name", name])
print("[k8s-run] k8s create exit=\(result.status)")
if result.status != 0 {
print("[k8s-run] k8s create stderr: \(result.error)")
dumpNodeDiagnostics(f, node: name)
}
try result.check()
#expect(result.output.contains(name))
let containerStatus = try f.getContainerStatus(name)
print("[k8s-run] container status=\(containerStatus)")
#expect(containerStatus == "running")
let kubeconfig = try loadKubeconfig()
let clusters = (kubeconfig["clusters"] as? [[String: Any]]) ?? []
#expect(clusters.contains { $0["name"] as? String == name })
let contexts = (kubeconfig["contexts"] as? [[String: Any]]) ?? []
#expect(contexts.contains { $0["name"] as? String == name })
let users = (kubeconfig["users"] as? [[String: Any]]) ?? []
#expect(users.contains { $0["name"] as? String == name })
}
}
@Test func testConcurrentCreateGetsDifferentPorts() async throws {
try await ContainerFixture.with { f in
let name1 = "k8s-\(f.testID)-a"
let name2 = "k8s-\(f.testID)-b"
f.addCleanup { _ = try? f.run(["k8s", "delete", "--name", name1]) }
f.addCleanup { _ = try? f.run(["k8s", "delete", "--name", name2]) }
print("[k8s-run] k8s create --name \(name1)")
let result1 = try f.run(["k8s", "create", "--name", name1])
print("[k8s-run] k8s create exit=\(result1.status)")
if result1.status != 0 {
print("[k8s-run] k8s create stderr: \(result1.error)")
dumpNodeDiagnostics(f, node: name1)
}
#expect(result1.status == 0)
print("[k8s-run] k8s create --name \(name2)")
let result2 = try f.run(["k8s", "create", "--name", name2])
print("[k8s-run] k8s create exit=\(result2.status)")
if result2.status != 0 {
print("[k8s-run] k8s create stderr: \(result2.error)")
dumpNodeDiagnostics(f, node: name2)
}
#expect(result2.status == 0)
#expect(try f.getContainerStatus(name1) == "running")
#expect(try f.getContainerStatus(name2) == "running")
let port1 = try f.inspectContainer(name1).configuration.publishedPorts
.first(where: { $0.containerPort == 6443 })?.hostPort
let port2 = try f.inspectContainer(name2).configuration.publishedPorts
.first(where: { $0.containerPort == 6443 })?.hostPort
print("[k8s-run] port1=\(port1.map(String.init) ?? "nil") port2=\(port2.map(String.init) ?? "nil")")
#expect(port1 != nil)
#expect(port2 != nil)
#expect(port1 != port2)
}
}
}
@@ -0,0 +1,190 @@
//===----------------------------------------------------------------------===//
// Copyright © 2026 Apple Inc. and the container project authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//===----------------------------------------------------------------------===//
import ContainerTestSupport
import Foundation
import SystemPackage
import Testing
import Yams
@Suite(.serialized)
struct TestK8sWriteConfigSerial {
private func dumpNodeDiagnostics(_ f: ContainerFixture, node: String) {
print("=== NODE DIAGNOSTICS [\(node)] ===")
let cmds: [(label: String, args: [String])] = [
("ip-link", ["ip", "link", "show"]),
("iptables-mss", ["iptables", "-t", "mangle", "-L", "-n", "-v"]),
("containerd", ["systemctl", "status", "containerd", "--no-pager", "-l"]),
("kubelet-log", ["journalctl", "-u", "kubelet", "--no-pager", "-n", "60"]),
("crictl-images", ["crictl", "images"]),
]
for (label, args) in cmds {
if let r = try? f.run(["exec", node] + args) {
let out = r.output.trimmingCharacters(in: .whitespacesAndNewlines)
let err = r.error.trimmingCharacters(in: .whitespacesAndNewlines)
print("[\(label)] exit=\(r.status)")
if !out.isEmpty { print(out) }
if !err.isEmpty { print("stderr: \(err)") }
}
}
print("=== END NODE DIAGNOSTICS ===")
}
private func kubeconfigPath() -> FilePath {
FilePath(FileManager.default.homeDirectoryForCurrentUser.path)
.appending(".kube")
.appending("config")
}
private func loadKubeconfig() throws -> [String: Any] {
let yaml = try String(contentsOfFile: kubeconfigPath().string, encoding: .utf8)
guard let parsed = try Yams.load(yaml: yaml) as? [String: Any] else {
throw CommandError.executionFailed("could not parse kubeconfig")
}
return parsed
}
private func dumpKubeconfig(for name: String) {
let path = kubeconfigPath()
guard FileManager.default.fileExists(atPath: path.string) else {
print("[k8s-cfg] \(path.lastComponent?.string ?? "config") does not exist")
return
}
guard let kubeconfig = try? loadKubeconfig() else {
print("[k8s-cfg] \(path.lastComponent?.string ?? "config") could not be parsed")
return
}
let clusters = (kubeconfig["clusters"] as? [[String: Any]])?.compactMap { $0["name"] as? String } ?? []
let contexts = (kubeconfig["contexts"] as? [[String: Any]])?.compactMap { $0["name"] as? String } ?? []
let current = kubeconfig["current-context"] as? String ?? "<none>"
print("[k8s-cfg] kubeconfig: clusters=\(clusters) contexts=\(contexts) current-context=\(current)")
}
private func dumpEnv(_ f: ContainerFixture, clusterName: String) {
print("=== ENV DUMP [\(clusterName)] ===")
if let result = try? f.run(["system", "status"]) {
print("[system status]\n\(result.output)")
}
if let result = try? f.run(["list"]) {
print("[container list]\n\(result.output)")
}
if let result = try? f.run(["inspect", clusterName]) {
print("[inspect \(clusterName)] stdout: \(result.output) stderr: \(result.error)")
}
print("=== END ENV DUMP ===")
}
private func kubeconfigContains(name: String) -> Bool {
guard let kubeconfig = try? loadKubeconfig() else { return false }
let clusters = (kubeconfig["clusters"] as? [[String: Any]]) ?? []
return clusters.contains { $0["name"] as? String == name }
}
@Test func testWriteConfigMergesContext() async throws {
try await ContainerFixture.with { f in
let name = "k8s-\(f.testID)"
f.addCleanup { _ = try? f.run(["k8s", "delete", "--name", name]) }
print("[k8s-cfg] k8s create --name \(name)")
let result = try f.run(["k8s", "create", "--name", name])
print("[k8s-cfg] k8s create exit=\(result.status)")
if result.status != 0 {
print("[k8s-cfg] k8s create stderr: \(result.error)")
dumpEnv(f, clusterName: name)
dumpNodeDiagnostics(f, node: name)
}
try result.check()
print("[k8s-cfg] k8s write-config --name \(name)")
let writeResult = try f.run(["k8s", "write-config", "--name", name])
print("[k8s-cfg] k8s write-config exit=\(writeResult.status)")
if writeResult.status != 0 {
print("[k8s-cfg] k8s write-config stderr: \(writeResult.error)")
dumpEnv(f, clusterName: name)
}
#expect(writeResult.status == 0)
let kubeconfig = try loadKubeconfig()
let clusters = (kubeconfig["clusters"] as? [[String: Any]]) ?? []
#expect(clusters.contains { $0["name"] as? String == name })
let contexts = (kubeconfig["contexts"] as? [[String: Any]]) ?? []
#expect(contexts.contains { $0["name"] as? String == name })
let users = (kubeconfig["users"] as? [[String: Any]]) ?? []
#expect(users.contains { $0["name"] as? String == name })
}
}
@Test func testWriteConfigIsIdempotent() async throws {
try await ContainerFixture.with { f in
let name = "k8s-\(f.testID)"
f.addCleanup { _ = try? f.run(["k8s", "delete", "--name", name]) }
print("[k8s-cfg] k8s create --name \(name)")
let result = try f.run(["k8s", "create", "--name", name])
print("[k8s-cfg] k8s create exit=\(result.status)")
if result.status != 0 {
print("[k8s-cfg] k8s create stderr: \(result.error)")
dumpEnv(f, clusterName: name)
dumpNodeDiagnostics(f, node: name)
}
try result.check()
print("[k8s-cfg] k8s write-config --name \(name) (first)")
_ = try f.run(["k8s", "write-config", "--name", name])
print("[k8s-cfg] k8s write-config --name \(name) (second)")
let secondResult = try f.run(["k8s", "write-config", "--name", name])
print("[k8s-cfg] k8s write-config (second) exit=\(secondResult.status)")
#expect(secondResult.status == 0)
let kubeconfig = try loadKubeconfig()
let clusters = (kubeconfig["clusters"] as? [[String: Any]]) ?? []
let matchingClusters = clusters.filter { $0["name"] as? String == name }
#expect(matchingClusters.count == 1)
}
}
@Test func testDeleteClusterCleansUpKubeconfig() async throws {
try await ContainerFixture.with { f in
let name = "k8s-\(f.testID)"
f.addCleanup { _ = try? f.run(["k8s", "delete", "--name", name]) }
print("[k8s-cfg] k8s create --name \(name)")
let result = try f.run(["k8s", "create", "--name", name])
print("[k8s-cfg] k8s create exit=\(result.status)")
if result.status != 0 {
print("[k8s-cfg] k8s create stderr: \(result.error)")
dumpEnv(f, clusterName: name)
dumpNodeDiagnostics(f, node: name)
}
try result.check()
print("[k8s-cfg] k8s write-config --name \(name)")
_ = try f.run(["k8s", "write-config", "--name", name])
#expect(kubeconfigContains(name: name))
print("[k8s-cfg] k8s delete --name \(name)")
let deleteResult = try f.run(["k8s", "delete", "--name", name])
print("[k8s-cfg] k8s delete exit=\(deleteResult.status) stderr=\(deleteResult.error)")
#expect(deleteResult.status == 0)
#expect(!kubeconfigContains(name: name))
}
}
}
+277
View File
@@ -0,0 +1,277 @@
//===----------------------------------------------------------------------===//
// Copyright © 2026 Apple Inc. and the container project authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//===----------------------------------------------------------------------===//
import ContainerResource
import Foundation
import Testing
import Yams
@testable import k8s
// MARK: - Fixtures
private let fixtureDefaultCPUs: Int = max(ProcessInfo.processInfo.processorCount / 4, 2)
private let fixtureDefaultMemoryMiB: UInt64 = {
let gb = max(Int(ProcessInfo.processInfo.physicalMemory / (1024 * 1024 * 1024)) / 4, 2)
return UInt64(gb * 1024)
}()
private func makeSnapshot(
id: String,
role: String,
status: RuntimeStatus = .running,
cpus: Int = 2,
memoryMiB: UInt64 = 2048,
addr: String = ""
) throws -> ContainerSnapshot {
let labelsJSON = #"{"com.apple.container.plugin":"k8s","com.apple.container.resource.role":"\#(role)"}"#
let networksJSON: String
if addr.isEmpty {
networksJSON = "[]"
} else {
networksJSON = #"[{"network":"bridge","hostname":"\#(id)","ipv4Address":"\#(addr)/24","ipv4Gateway":"10.0.0.254"}]"#
}
let json = """
{
"configuration": {
"id": "\(id)",
"image": {
"reference": "docker.io/kindest/node:v1.35.5",
"descriptor": {"mediaType":"","digest":"sha256:abc","size":0}
},
"initProcess": {"executable":"/bin/sh","arguments":[],"environment":[],"workingDirectory":"/","terminal":false,"user":{"id":{"uid":0,"gid":0}},"supplementalGroups":[],"rlimits":[]},
"resources": {"cpus":\(cpus),"memoryInBytes":\(memoryMiB * 1024 * 1024)},
"labels": \(labelsJSON)
},
"status": "\(status.rawValue)",
"networks": \(networksJSON)
}
"""
return try JSONDecoder().decode(ContainerSnapshot.self, from: Data(json.utf8))
}
private func makeControlPlane(_ name: String, cpus: Int = fixtureDefaultCPUs, memoryMiB: UInt64 = fixtureDefaultMemoryMiB, addr: String = "") throws -> ContainerSnapshot {
try makeSnapshot(id: name, role: K8sHelper.controlPlaneRoleName, cpus: cpus, memoryMiB: memoryMiB, addr: addr)
}
private func makeWorker(_ id: String, cpus: Int = max(fixtureDefaultCPUs / 2, 1), memoryMiB: UInt64 = max(fixtureDefaultMemoryMiB / 2, 512)) throws -> ContainerSnapshot {
try makeSnapshot(id: id, role: "worker", cpus: cpus, memoryMiB: memoryMiB)
}
// MARK: - K8sNodeResource header / row shape
@Suite("K8sNodeResource")
struct K8sNodeRowTests {
@Test func tableHeaderHasEightColumns() throws {
#expect(K8sNodeResource.tableHeader.count == 8)
}
@Test func tableHeaderLabels() throws {
#expect(K8sNodeResource.tableHeader == ["CLUSTER", "NODE", "ROLE", "STATE", "CPUS", "MEMORY", "ADDR", "PORTS"])
}
@Test func rowColumnCountMatchesHeader() throws {
let snapshot = try makeControlPlane("dev")
let row = K8sNodeResource(clusterName: "dev", snapshot: snapshot)
#expect(row.tableRow.count == K8sNodeResource.tableHeader.count)
}
@Test func controlPlaneRowValues() throws {
let cpus = fixtureDefaultCPUs
let memoryMiB = fixtureDefaultMemoryMiB
let snapshot = try makeControlPlane("dev", cpus: cpus, memoryMiB: memoryMiB, addr: "10.0.0.1")
let row = K8sNodeResource(clusterName: "dev", snapshot: snapshot)
let cols = row.tableRow
#expect(cols[0] == "dev") // CLUSTER
#expect(cols[1] == "dev") // NODE
#expect(cols[2] == K8sHelper.controlPlaneRoleName) // ROLE
#expect(cols[3] == "running") // STATE
#expect(cols[4] == "\(cpus)") // CPUS
#expect(cols[5] == "\(memoryMiB) MB") // MEMORY
#expect(cols[6].contains("10.0.0.1")) // ADDR
// cols[7] PORTS: no publishedPorts in fixture empty string
#expect(cols[7] == "")
}
@Test func workerRowValues() throws {
let cpus = max(fixtureDefaultCPUs / 2, 1)
let memoryMiB = max(fixtureDefaultMemoryMiB / 2, 512)
let snapshot = try makeWorker("dev-worker-1", cpus: cpus, memoryMiB: memoryMiB)
let row = K8sNodeResource(clusterName: "dev", snapshot: snapshot)
let cols = row.tableRow
#expect(cols[0] == "dev") // CLUSTER
#expect(cols[1] == "dev-worker-1") // NODE
#expect(cols[2] == "worker") // ROLE
#expect(cols[4] == "\(cpus)") // CPUS
#expect(cols[5] == "\(memoryMiB) MB") // MEMORY
}
@Test func quietValueIsNodeID() throws {
let snapshot = try makeWorker("dev-worker-2")
let row = K8sNodeResource(clusterName: "dev", snapshot: snapshot)
#expect(row.quietValue == "dev-worker-2")
}
}
// MARK: - buildK8sRows ordering and grouping
@Suite("K8sHelper.buildK8sRows")
struct BuildK8sRowsTests {
@Test func emptyInputProducesNoRows() {
#expect(K8sHelper.buildK8sRows(from: []).isEmpty)
}
@Test func singleControlPlaneAlone() throws {
let cp = try makeControlPlane("dev")
let rows = K8sHelper.buildK8sRows(from: [cp])
#expect(rows.count == 1)
#expect(rows[0].snapshot.configuration.id == "dev")
}
@Test func controlPlaneFollowedByItsWorkers() throws {
let cp = try makeControlPlane("dev")
let w1 = try makeWorker("dev-worker-1")
let w2 = try makeWorker("dev-worker-2")
let rows = K8sHelper.buildK8sRows(from: [w2, w1, cp])
#expect(rows.count == 3)
#expect(rows[0].snapshot.id == "dev")
#expect(rows[1].snapshot.id == "dev-worker-1")
#expect(rows[2].snapshot.id == "dev-worker-2")
}
@Test func workersGroupedUnderCorrectControlPlane() throws {
let cp1 = try makeControlPlane("alpha")
let cp2 = try makeControlPlane("beta")
let w1 = try makeWorker("alpha-worker-1")
let w2 = try makeWorker("beta-worker-1")
let rows = K8sHelper.buildK8sRows(from: [w2, cp2, w1, cp1])
#expect(rows[0].snapshot.id == "alpha")
#expect(rows[1].snapshot.id == "alpha-worker-1")
#expect(rows[1].clusterName == "alpha")
#expect(rows[2].snapshot.id == "beta")
#expect(rows[3].snapshot.id == "beta-worker-1")
#expect(rows[3].clusterName == "beta")
}
@Test func multipleClustersAreSortedByName() throws {
let cpZ = try makeControlPlane("zebra")
let cpA = try makeControlPlane("apple")
let rows = K8sHelper.buildK8sRows(from: [cpZ, cpA])
#expect(rows[0].snapshot.id == "apple")
#expect(rows[1].snapshot.id == "zebra")
}
@Test func orphanedWorkerAppearsAtEnd() throws {
let cp = try makeControlPlane("dev")
let orphan = try makeWorker("old-worker-1")
let rows = K8sHelper.buildK8sRows(from: [orphan, cp])
#expect(rows.count == 2)
#expect(rows[0].snapshot.id == "dev")
#expect(rows[1].snapshot.id == "old-worker-1")
#expect(rows[1].clusterName == "old")
}
@Test func orphanedWorkerClusterNameDerivedFromID() throws {
let orphan = try makeWorker("mycluster-worker-3")
let rows = K8sHelper.buildK8sRows(from: [orphan])
#expect(rows.count == 1)
#expect(rows[0].clusterName == "mycluster")
}
@Test func clusterNameWithWorkerInItIsHandledCorrectly() throws {
let cp = try makeControlPlane("foo-worker")
let w = try makeWorker("foo-worker-worker-1")
let rows = K8sHelper.buildK8sRows(from: [w, cp])
#expect(rows.count == 2)
#expect(rows[0].snapshot.id == "foo-worker")
#expect(rows[1].snapshot.id == "foo-worker-worker-1")
#expect(rows[1].clusterName == "foo-worker")
}
@Test func workerNotAssignedToWrongCluster() throws {
let cp = try makeControlPlane("dev")
let w = try makeWorker("dev2-worker-1")
let rows = K8sHelper.buildK8sRows(from: [w, cp])
#expect(rows[0].snapshot.id == "dev")
#expect(rows[1].snapshot.id == "dev2-worker-1")
#expect(rows[1].clusterName == "dev2")
}
}
// MARK: - K8sHelper.fqdn
@Suite("K8sHelper.fqdn")
struct FQDNTests {
@Test func nilDomainReturnsNil() {
#expect(K8sHelper.fqdn(for: "dev", domain: nil) == nil)
}
@Test func emptyDomainReturnsNil() {
#expect(K8sHelper.fqdn(for: "dev", domain: "") == nil)
}
@Test func nameWithDotReturnedAsIs() {
#expect(K8sHelper.fqdn(for: "dev.local", domain: "example.com") == "dev.local")
}
@Test func plainNameGetsDomainAppended() {
#expect(K8sHelper.fqdn(for: "dev", domain: "example.com") == "dev.example.com")
}
}
// MARK: - KubeConfig
@Suite("KubeConfig")
struct KubeKubeConfigTests {
private func sampleYAML(clusterName: String = "default") -> String {
"""
apiVersion: v1
kind: Config
clusters:
- name: \(clusterName)
cluster:
server: https://127.0.0.1:6443
certificate-authority-data: dGVzdA==
contexts:
- name: \(clusterName)
context:
cluster: \(clusterName)
user: \(clusterName)
current-context: \(clusterName)
users:
- name: \(clusterName)
user:
client-certificate-data: dGVzdA==
client-key-data: dGVzdA==
"""
}
@Test func decodeRoundTrip() throws {
let config = try YAMLDecoder().decode(KubeConfig.self, from: sampleYAML())
#expect(config.apiVersion == "v1")
#expect(config.kind == "Config")
#expect(config.clusters.count == 1)
#expect(config.clusters[0].cluster.server == "https://127.0.0.1:6443")
#expect(config.contexts.count == 1)
#expect(config.users.count == 1)
#expect(config.currentContext == "default")
}
@Test func emptyConfigEncodesWithoutError() throws {
let output = try YAMLEncoder().encode(KubeConfig.empty)
#expect(!output.isEmpty)
}
}
@@ -0,0 +1,494 @@
//===----------------------------------------------------------------------===//
// Copyright © 2026 Apple Inc. and the container project authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//===----------------------------------------------------------------------===//
import Darwin
import Foundation
import Logging
import SystemPackage
import Testing
import Yams
@testable import k8s
// MARK: - Helpers
private func makeTempFile() throws -> (FilePath, cleanup: () -> Void) {
let path = FilePath(FileManager.default.temporaryDirectory.path)
.appending("kubeconfig-test-\(UUID().uuidString)")
return (path, { try? FileManager.default.removeItem(atPath: path.string) })
}
private func decode(_ yaml: String) throws -> KubeConfig {
try YAMLDecoder().decode(KubeConfig.self, from: yaml)
}
private func encode(_ config: KubeConfig) throws -> String {
try YAMLEncoder().encode(config)
}
private let log = Logger(label: "test")
// MARK: - Round-trip fidelity
@Suite("KubeConfig round-trip fidelity")
struct KubeconfigRoundTripTests {
@Test func execAuthRoundTrip() throws {
let yaml = """
apiVersion: v1
kind: Config
clusters:
- name: gke-cluster
cluster:
server: https://1.2.3.4
certificate-authority-data: dGVzdA==
contexts:
- name: gke-context
context:
cluster: gke-cluster
user: gke-user
namespace: production
current-context: gke-context
users:
- name: gke-user
user:
exec:
apiVersion: client.authentication.k8s.io/v1beta1
command: gke-gcloud-auth-plugin
args:
- --some-flag
env:
- name: USE_GKE_GCLOUD_AUTH_PLUGIN
value: "True"
installHint: Install gke-gcloud-auth-plugin
provideClusterInfo: true
interactiveMode: IfAvailable
"""
let config = try decode(yaml)
let user = try #require(config.users.first)
let exec = try #require(user.authInfo.exec)
#expect(exec.command == "gke-gcloud-auth-plugin")
#expect(exec.args == ["--some-flag"])
#expect(exec.env?.first?.name == "USE_GKE_GCLOUD_AUTH_PLUGIN")
#expect(exec.env?.first?.value == "True")
#expect(exec.installHint == "Install gke-gcloud-auth-plugin")
#expect(exec.provideClusterInfo == true)
#expect(exec.interactiveMode == "IfAvailable")
// Re-encode and decode again fields must survive the round-trip
let reencoded = try encode(config)
let redecoded = try decode(reencoded)
let exec2 = try #require(redecoded.users.first?.authInfo.exec)
#expect(exec2.command == exec.command)
#expect(exec2.args == exec.args)
#expect(exec2.env?.first?.name == exec.env?.first?.name)
#expect(exec2.provideClusterInfo == exec.provideClusterInfo)
#expect(redecoded.contexts.first?.context.namespace == "production")
}
@Test func tokenAuthRoundTrip() throws {
let yaml = """
apiVersion: v1
kind: Config
clusters:
- name: my-cluster
cluster:
server: https://1.2.3.4
contexts:
- name: my-context
context:
cluster: my-cluster
user: my-user
current-context: my-context
users:
- name: my-user
user:
token: supersecrettoken
"""
let config = try decode(yaml)
#expect(config.users.first?.authInfo.token == "supersecrettoken")
let redecoded = try decode(try encode(config))
#expect(redecoded.users.first?.authInfo.token == "supersecrettoken")
}
@Test func authProviderRoundTrip() throws {
let yaml = """
apiVersion: v1
kind: Config
clusters:
- name: legacy-gke
cluster:
server: https://1.2.3.4
contexts:
- name: legacy-context
context:
cluster: legacy-gke
user: legacy-user
current-context: legacy-context
users:
- name: legacy-user
user:
auth-provider:
name: gcp
config:
cmd-path: /usr/lib/google-cloud-sdk/bin/gcloud
token-key: '{.credential.access_token}'
"""
let config = try decode(yaml)
let provider = try #require(config.users.first?.authInfo.authProvider)
#expect(provider.name == "gcp")
#expect(provider.config?["cmd-path"] == "/usr/lib/google-cloud-sdk/bin/gcloud")
let redecoded = try decode(try encode(config))
let provider2 = try #require(redecoded.users.first?.authInfo.authProvider)
#expect(provider2.name == provider.name)
#expect(provider2.config?["cmd-path"] == provider.config?["cmd-path"])
}
@Test func clusterFieldsRoundTrip() throws {
let yaml = """
apiVersion: v1
kind: Config
clusters:
- name: proxied-cluster
cluster:
server: https://1.2.3.4
tls-server-name: my-server.example.com
insecure-skip-tls-verify: true
proxy-url: http://proxy.example.com:8080
contexts:
- name: proxied-context
context:
cluster: proxied-cluster
user: proxied-user
current-context: proxied-context
users:
- name: proxied-user
user:
token: abc
"""
let config = try decode(yaml)
let cluster = try #require(config.clusters.first?.cluster)
#expect(cluster.tlsServerName == "my-server.example.com")
#expect(cluster.insecureSkipTLSVerify == true)
#expect(cluster.proxyURL == "http://proxy.example.com:8080")
let redecoded = try decode(try encode(config))
let cluster2 = try #require(redecoded.clusters.first?.cluster)
#expect(cluster2.tlsServerName == cluster.tlsServerName)
#expect(cluster2.insecureSkipTLSVerify == cluster.insecureSkipTLSVerify)
#expect(cluster2.proxyURL == cluster.proxyURL)
}
}
// MARK: - mergeConfig behavior
@Suite("K8sHelper.mergeConfig")
struct MergeConfigTests {
private func makeConfig(clusterName: String, server: String = "https://127.0.0.1:6443") -> KubeConfig {
var config = KubeConfig()
config.clusters = [NamedCluster(name: clusterName, cluster: Cluster(server: server, certificateAuthorityData: "dGVzdA=="))]
config.contexts = [NamedContext(name: clusterName, context: Context(cluster: clusterName, user: clusterName))]
config.users = [NamedAuthInfo(name: clusterName, authInfo: AuthInfo(clientCertificateData: "dGVzdA==", clientKeyData: "dGVzdA=="))]
config.currentContext = clusterName
return config
}
@Test func mergeIntoEmptyFileCreatesFile() throws {
let (path, cleanup) = try makeTempFile()
defer { cleanup() }
try K8sHelper.mergeConfig(makeConfig(clusterName: "dev"), containerId: "dev", targetPath: path, setCurrentContext: true, log: log)
let written = try decode(String(contentsOfFile: path.string, encoding: .utf8))
#expect(written.clusters.count == 1)
#expect(written.clusters[0].name == "dev")
#expect(written.currentContext == "dev")
}
@Test func mergePreservesExistingExecAuthEntry() throws {
let (path, cleanup) = try makeTempFile()
defer { cleanup() }
// Write an existing kubeconfig with a GKE exec-auth entry
let existingYAML = """
apiVersion: v1
kind: Config
clusters:
- name: gke-prod
cluster:
server: https://5.6.7.8
certificate-authority-data: dGVzdA==
contexts:
- name: gke-prod
context:
cluster: gke-prod
user: gke-prod-user
namespace: production
current-context: gke-prod
users:
- name: gke-prod-user
user:
exec:
apiVersion: client.authentication.k8s.io/v1beta1
command: gke-gcloud-auth-plugin
provideClusterInfo: true
interactiveMode: IfAvailable
"""
try existingYAML.write(toFile: path.string, atomically: true, encoding: .utf8)
try K8sHelper.mergeConfig(makeConfig(clusterName: "dev"), containerId: "dev", targetPath: path, log: log)
let result = try decode(String(contentsOfFile: path.string, encoding: .utf8))
// New entry present
#expect(result.clusters.contains { $0.name == "dev" })
#expect(result.currentContext == "gke-prod") // existing context preserved not overwritten
// GKE entry fully preserved
let gkeCluster = try #require(result.clusters.first { $0.name == "gke-prod" })
#expect(gkeCluster.cluster.server == "https://5.6.7.8")
let gkeUser = try #require(result.users.first { $0.name == "gke-prod-user" })
let exec = try #require(gkeUser.authInfo.exec)
#expect(exec.command == "gke-gcloud-auth-plugin")
#expect(exec.provideClusterInfo == true)
#expect(exec.interactiveMode == "IfAvailable")
let gkeContext = try #require(result.contexts.first { $0.name == "gke-prod" })
#expect(gkeContext.context.namespace == "production")
}
@Test func mergePreservesTokenAuthEntry() throws {
let (path, cleanup) = try makeTempFile()
defer { cleanup() }
let existingYAML = """
apiVersion: v1
kind: Config
clusters:
- name: staging
cluster:
server: https://9.10.11.12
contexts:
- name: staging
context:
cluster: staging
user: staging-user
current-context: staging
users:
- name: staging-user
user:
token: verysecrettoken
"""
try existingYAML.write(toFile: path.string, atomically: true, encoding: .utf8)
try K8sHelper.mergeConfig(makeConfig(clusterName: "dev"), containerId: "dev", targetPath: path, log: log)
let result = try decode(String(contentsOfFile: path.string, encoding: .utf8))
let stagingUser = try #require(result.users.first { $0.name == "staging-user" })
#expect(stagingUser.authInfo.token == "verysecrettoken")
}
@Test func mergeReplacesExistingEntryWithSameName() throws {
let (path, cleanup) = try makeTempFile()
defer { cleanup() }
try K8sHelper.mergeConfig(makeConfig(clusterName: "dev", server: "https://127.0.0.1:6445"), containerId: "dev", targetPath: path, log: log)
try K8sHelper.mergeConfig(makeConfig(clusterName: "dev", server: "https://127.0.0.1:6446"), containerId: "dev", targetPath: path, log: log)
let result = try decode(String(contentsOfFile: path.string, encoding: .utf8))
#expect(result.clusters.filter { $0.name == "dev" }.count == 1)
#expect(result.contexts.filter { $0.name == "dev" }.count == 1)
#expect(result.users.filter { $0.name == "dev" }.count == 1)
#expect(result.clusters.first { $0.name == "dev" }?.cluster.server == "https://127.0.0.1:6446")
}
@Test func mergeThrowsWhenExistingFileIsInvalidYAML() throws {
let (path, cleanup) = try makeTempFile()
defer { cleanup() }
try "this: is: not: valid: yaml: [[[".write(toFile: path.string, atomically: true, encoding: .utf8)
#expect(throws: (any Error).self) {
try K8sHelper.mergeConfig(makeConfig(clusterName: "dev"), containerId: "dev", targetPath: path, log: log)
}
}
@Test func mergeSetsCurrentContextOnFirstClusterOnly() throws {
let (path, cleanup) = try makeTempFile()
defer { cleanup() }
// First create: sets current-context because kubeconfig is empty
try K8sHelper.mergeConfig(makeConfig(clusterName: "first"), containerId: "first", targetPath: path, setCurrentContext: true, log: log)
// Second create: does NOT overwrite current-context because it's already set
try K8sHelper.mergeConfig(makeConfig(clusterName: "second"), containerId: "second", targetPath: path, setCurrentContext: true, log: log)
let result = try decode(String(contentsOfFile: path.string, encoding: .utf8))
#expect(result.currentContext == "first") // first-use semantics: not overwritten by second cluster
#expect(result.clusters.count == 2)
}
}
// MARK: - env-dependent tests
// Both suites mutate the KUBECONFIG environment variable; wrap them in a
// common .serialized parent so they cannot race against each other.
@Suite("K8sHelper env-dependent", .serialized)
struct KubeconfigEnvTests {
// MARK: - resolveKubeconfigMergePath
@Suite("K8sHelper.resolveKubeconfigMergePath")
struct ResolveKubeconfigMergePathTests {
private func withKubeconfigEnv(_ value: String?, _ body: () -> Void) {
let key = "KUBECONFIG"
let original = Darwin.getenv(key).map { String(cString: $0) }
if let value {
setenv(key, value, 1)
} else {
unsetenv(key)
}
body()
if let original {
setenv(key, original, 1)
} else {
unsetenv(key)
}
}
@Test func noEnvDefaultsToHomeKubeConfig() {
withKubeconfigEnv(nil) {
let path = K8sHelper.resolveKubeconfigMergePath()
#expect(path.string.hasSuffix(".kube/config"))
}
}
@Test func singleEnvPathUsedDirectly() {
withKubeconfigEnv("/tmp/my-kubeconfig") {
let path = K8sHelper.resolveKubeconfigMergePath()
#expect(path.string == "/tmp/my-kubeconfig")
}
}
@Test func multiplePathsFirstExistingWins() throws {
let existing = FilePath(FileManager.default.temporaryDirectory.path)
.appending("kube-exists-\(UUID().uuidString)")
try "".write(toFile: existing.string, atomically: true, encoding: .utf8)
defer { try? FileManager.default.removeItem(atPath: existing.string) }
withKubeconfigEnv("/tmp/does-not-exist-a:\(existing.string):/tmp/does-not-exist-b") {
let path = K8sHelper.resolveKubeconfigMergePath()
#expect(path.string == existing.string)
}
}
@Test func multiplePathsNoneExistUsesLast() {
withKubeconfigEnv("/tmp/no-exist-a:/tmp/no-exist-b:/tmp/no-exist-c") {
let path = K8sHelper.resolveKubeconfigMergePath()
#expect(path.string == "/tmp/no-exist-c")
}
}
}
// MARK: - removeConfig behavior
@Suite("K8sHelper.removeConfig")
struct RemoveConfigTests {
private func makeConfig(clusterName: String) -> KubeConfig {
var config = KubeConfig()
config.clusters = [NamedCluster(name: clusterName, cluster: Cluster(server: "https://127.0.0.1:6443"))]
config.contexts = [NamedContext(name: clusterName, context: Context(cluster: clusterName, user: clusterName))]
config.users = [NamedAuthInfo(name: clusterName, authInfo: AuthInfo())]
return config
}
private func withKubeconfig(_ initial: KubeConfig, _ body: () throws -> Void) throws {
let tmp = FilePath(FileManager.default.temporaryDirectory.path)
.appending("kubeconfig-remove-\(UUID().uuidString)")
defer { try? FileManager.default.removeItem(atPath: tmp.string) }
try encode(initial).write(toFile: tmp.string, atomically: true, encoding: .utf8)
let original = Darwin.getenv("KUBECONFIG").map { String(cString: $0) }
setenv("KUBECONFIG", tmp.string, 1)
defer {
if let original { setenv("KUBECONFIG", original, 1) } else { unsetenv("KUBECONFIG") }
}
try body()
}
private func currentKubeconfig() throws -> KubeConfig {
let path = K8sHelper.resolveKubeconfigMergePath()
return try decode(String(contentsOfFile: path.string, encoding: .utf8))
}
@Test func removesClusterContextAndUserEntries() throws {
var initial = makeConfig(clusterName: "dev")
let other = makeConfig(clusterName: "other")
initial.clusters += other.clusters
initial.contexts += other.contexts
initial.users += other.users
initial.currentContext = "other"
try withKubeconfig(initial) {
try K8sHelper.removeConfig(containerId: "dev", log: log)
let result = try currentKubeconfig()
#expect(result.clusters.count == 1)
#expect(!result.clusters.contains { $0.name == "dev" })
#expect(result.users.count == 1)
#expect(result.contexts.count == 1)
}
}
@Test func clearsCurrentContextWhenItMatchesDeletedCluster() throws {
var initial = makeConfig(clusterName: "dev")
initial.currentContext = "dev"
try withKubeconfig(initial) {
try K8sHelper.removeConfig(containerId: "dev", log: log)
#expect(try currentKubeconfig().currentContext == nil)
}
}
@Test func preservesCurrentContextWhenItPointsElsewhere() throws {
var initial = makeConfig(clusterName: "dev")
let other = makeConfig(clusterName: "other")
initial.clusters += other.clusters
initial.contexts += other.contexts
initial.users += other.users
initial.currentContext = "other"
try withKubeconfig(initial) {
try K8sHelper.removeConfig(containerId: "dev", log: log)
#expect(try currentKubeconfig().currentContext == "other")
}
}
@Test func noopWhenFileDoesNotExist() throws {
let missing = FilePath(FileManager.default.temporaryDirectory.path)
.appending("kubeconfig-missing-\(UUID().uuidString)")
let original = Darwin.getenv("KUBECONFIG").map { String(cString: $0) }
setenv("KUBECONFIG", missing.string, 1)
defer {
if let original { setenv("KUBECONFIG", original, 1) } else { unsetenv("KUBECONFIG") }
}
try K8sHelper.removeConfig(containerId: "dev", log: log)
}
}
}
+172
View File
@@ -1583,3 +1583,175 @@ container system property list
# output as JSON for scripting
container system property list --format json
```
## Kubernetes Cluster Management
`container k8s` manages local single-node Kubernetes clusters backed by container VMs. Each cluster runs a Kubernetes control-plane node inside a container using `kindest/node` and `kubeadm`.
> [!IMPORTANT]
> The `k8s` command is an experimental feature and its subcommands and options are subject to change.
### `container k8s create`
Creates and starts a local Kubernetes cluster. Pulls the node image if needed, runs `kubeadm init`, installs the kindnet CNI, and merges the cluster credentials into `~/.kube/config`.
**Usage**
```bash
container k8s create [--name <name>] [--node-image <image>] [--rm] [<resource options>] [--debug]
```
**Options**
* `--name <name>`: Cluster name (default: `k8s-dev`)
* `--node-image <image>`: Node image reference (default: `docker.io/kindest/node:v1.35.5`)
* `--rm`: Remove the cluster container after it stops
**Resource Options**
* `--cpus <cpus>`: Number of virtual CPUs (default: 1/4 of host CPUs, minimum 2)
* `--memory <memory>`: Memory allocation (default: 1/4 of host memory, minimum 2g)
**Registry Options**
* `--scheme <scheme>`: Scheme for the container registry (values: http, https, auto; default: auto)
**Image Fetch Options**
* `--max-concurrent-downloads <n>`: Maximum number of concurrent downloads (default: 3)
**Examples**
```bash
# create a cluster with the default name (k8s-dev)
container k8s create
# create a cluster with a custom name and resource allocation
container k8s create --name my-cluster --cpus 4 --memory 8g
# create a cluster that removes itself when stopped
container k8s create --name temp-cluster --rm
```
### `container k8s start`
Starts a stopped Kubernetes cluster and refreshes its entry in `~/.kube/config` (the container IP can change between starts).
**Usage**
```bash
container k8s start [--name <name>] [--debug]
```
**Options**
* `--name <name>`: Cluster name (default: `k8s-dev`)
**Examples**
```bash
# start the default cluster
container k8s start
# start a named cluster
container k8s start --name my-cluster
```
### `container k8s delete (rm)`
Stops and deletes a Kubernetes cluster container and removes its entry from `~/.kube/config`.
**Usage**
```bash
container k8s delete [--name <name>] [--debug]
```
**Options**
* `--name <name>`: Cluster name (default: `k8s-dev`)
**Examples**
```bash
# delete the default cluster
container k8s delete
# delete a named cluster
container k8s delete --name my-cluster
container k8s rm --name my-cluster
```
### `container k8s list (ls)`
Lists all Kubernetes clusters with their status and node image.
**Usage**
```bash
container k8s list [--debug]
```
**Examples**
```bash
container k8s list
container k8s ls
```
### `container k8s load-image`
Exports an image from the local `container` image store and imports it into the cluster's containerd (in the `k8s.io` namespace) so that Kubernetes can schedule pods that reference it.
**Usage**
```bash
container k8s load-image [--name <name>] [--platform <platform>] <image> [--debug]
```
**Arguments**
* `<image>`: Image reference to load (e.g. `my-app:latest`)
**Options**
* `--name <name>`: Cluster name (default: `k8s-dev`)
* `--platform <platform>`: Platform of the image variant to load from a multi-arch image (format: os/arch[/variant], default: `linux/<host-arch>`). Use this when the local store contains a multi-arch manifest list and you want to select a specific variant.
**Examples**
```bash
# load an image into the default cluster
container k8s load-image my-app:latest
# load an image into a named cluster
container k8s load-image --name my-cluster my-app:latest
# load the amd64 variant of a multi-arch image
container k8s load-image --platform linux/amd64 my-app:latest
```
### `container k8s write-config`
Fetches the current kubeconfig from a running cluster and merges its context into a kubeconfig file. Use this to refresh credentials after a cluster restart or to write to an alternate config file.
**Usage**
```bash
container k8s write-config [--name <name>] [--kubeconfig <path>] [--debug]
```
**Options**
* `--name <name>`: Cluster name (default: `k8s-dev`)
* `--kubeconfig <path>`: Path to the kubeconfig file to write or append to (default: `~/.kube/config`)
**Examples**
```bash
# refresh credentials for the default cluster into ~/.kube/config
container k8s write-config
# write the context for a named cluster to an alternate kubeconfig file
container k8s write-config --name my-cluster --kubeconfig ~/.kube/my-cluster.kubeconfig
```