ci: run unit test shards in parallel (#2314)

* ci: run unit test shards in parallel

* fix(ci): address shard-selection review feedback

* ci: cache package downloads in the shard and dedicated test jobs

.github/workflows/pr-checks.yml

@@ -41,8 +41,8 @@ jobs:
       - name: Install dependencies
         run: bun install --frozen-lockfile
 
-      - name: Smoke and full unit test suite
-        run: bun run check
+      - name: Smoke and static checks
+        run: bun run check:static
 
       - name: Suspicious PR intent scan
         env:
@@ -50,6 +50,80 @@ jobs:
           PR_SCAN_HEAD: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || 'HEAD' }}
         run: bun run security:pr-scan -- --base "$PR_SCAN_BASE" --head "$PR_SCAN_HEAD"
 
+  unit-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [1, 2, 3, 4, 5, 6, 7, 8]
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Set up Node.js
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: "22"
+
+      - name: Set up Bun
+        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
+        with:
+          bun-version-file: .bun-version
+
+      - name: Cache Bun packages
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ~/.bun/install/cache
+          key: ${{ runner.os }}-bun-root-${{ hashFiles('bun.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-bun-root-
+
+      - name: Install dependencies
+        run: bun install --frozen-lockfile
+
+      - name: Build
+        run: bun run build
+
+      - name: Unit test shard
+        run: bun run scripts/run-tests.ts --shards=8 --shard-index=${{ matrix.shard }} --feature=UNATTENDED_RETRY --max-concurrency=1
+
+  dedicated-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Set up Node.js
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: "22"
+
+      - name: Set up Bun
+        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
+        with:
+          bun-version-file: .bun-version
+
+      - name: Cache Bun packages
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ~/.bun/install/cache
+          key: ${{ runner.os }}-bun-root-${{ hashFiles('bun.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-bun-root-
+
+      - name: Install dependencies
+        run: bun install --frozen-lockfile
+
+      - name: Conversation arc tests
+        run: bun run test:conversation-arc
+
+      - name: Context efficiency tests
+        run: bun run test:context-efficiency
+
   launcher-compatibility:
     runs-on: ubuntu-latest
     timeout-minutes: 5

CONTRIBUTING.md

@@ -306,11 +306,11 @@ bun run web:build
 
 Notes on the local preflight:
 
-- `bun run check` already builds the CLI and includes smoke, deadcode, and the full unit pass (`test:full`) — do not run those separately, or you execute work twice.
-- `bun run test` isolates each test file's globals and requires Bun's completed JUnit report as well as a successful exit. `test:full` runs all eight native Bun shards serially to release memory between groups, then applies the same completion check to the conversation-arc pass and the context-efficiency pass with `HISTORY_SNIP` enabled. Every shard must produce a complete report; a failed shard does not prevent the remaining shards from running. A test that terminates the runner early fails the command, even if it exits with code zero. Each shard has a ten-minute deadline (override with `--shard-timeout-ms=<milliseconds>`), reports its start/end and elapsed time, and kills its process group on timeout or cancellation. A timeout stops the remaining shards and exits nonzero. On Linux, shard output is relayed from temporary regular files to avoid Bun’s isolated-stdio pipe registration bug; diagnostics remain visible during execution. Temporary reports and logs are removed after the check. CI additionally caps the suite job at 15 minutes and the other jobs at five minutes.
+- `bun run check` runs `check:static` (the any-usage budget, smoke build, and deadcode checks), then the full unit pass (`test:full`) — do not run those separately, or you execute work twice.
+- `bun run test` isolates each test file's globals and requires Bun's completed JUnit report as well as a successful exit. `test:full` runs all eight native Bun shards serially to release memory between groups. Use `bun run scripts/run-tests.ts --shards=N --shard-index=K` with the same test flags to run only shard K of N. `test:full` then applies the same completion check to the conversation-arc pass and the context-efficiency pass with `HISTORY_SNIP` enabled. Every shard must produce a complete report; a failed shard does not prevent the remaining shards from running. A test that terminates the runner early fails the command, even if it exits with code zero. Each shard has a ten-minute deadline (override with `--shard-timeout-ms=<milliseconds>`), reports its start/end and elapsed time, and kills its process group on timeout or cancellation. A timeout stops the remaining shards and exits nonzero. On Linux, shard output is relayed from temporary regular files to avoid Bun’s isolated-stdio pipe registration bug; diagnostics remain visible during execution. Temporary reports and logs are removed after the check. CI runs `check:static` in `smoke-and-tests`, runs the eight unit shards as parallel `unit-tests` jobs, and runs each dedicated feature pass once in `dedicated-tests`. Each unit shard builds the CLI before testing. CI caps these jobs at 15 minutes each and the other jobs at five minutes.
 - Fetching upstream `main` by URL avoids assuming that a fork checkout's `origin` points at Twigpine/openclaude. `FETCH_HEAD` is the fetched upstream tip. The scan deliberately uses local `HEAD` so it includes commits that have not been pushed yet; CI uses the pushed PR head SHA after the push.
 - The web CI job remains unconditional as an integration backstop. Contributors do not need to run the web suite locally for changes that cannot affect the site.
-- This preflight covers the same command families as CI, but it does not reproduce CI exactly in one shell. CI runs the full suite once with Bun and Node 22 available for smoke checks, separately builds and checks launcher compatibility under Node 22 and 24.11.x, separately builds and launches under exact Node 22.0.0, and executes every job on a clean runner.
+- This preflight covers the same command families as CI, but it does not reproduce CI exactly in one shell. CI runs static checks with Node 22 available for smoke checks, runs the unit shards and dedicated feature passes under Node 22, separately builds and checks launcher compatibility under Node 22 and 24.11.x, separately builds and launches under exact Node 22.0.0, and executes every job on a clean runner.
 
 If a local or GitHub CI check fails, first determine whether the PR owns the failure. Treat a failure as pre-existing only when you can reproduce it on the PR's current base (or link to the same failure on a current `main` CI run) and the PR neither changes the causal surface nor claims to fix that problem.
 

package.json

@@ -68,7 +68,8 @@
     "deadcode": "knip --include files,dependencies",
     "lint": "eslint .",
     "lint:any-budget": "bun run scripts/any-usage-report.ts --check",
-    "check": "bun run lint:any-budget && bun run smoke && bun run deadcode && bun run test:full",
+    "check:static": "bun run lint:any-budget && bun run smoke && bun run deadcode",
+    "check": "bun run check:static && bun run test:full",
     "check:strict": "bun run typecheck && bun run lint && bun run lint:any-budget && bun run smoke && bun run deadcode && bun run test:full",
     "verify:privacy": "bun run scripts/verify-no-phone-home.ts",
     "install:verify": "bun run scripts/verify-clean-install.ts",

scripts/run-tests.test.ts

@@ -112,6 +112,102 @@ for (const shards of [1, 2]) {
   }
 }
 
+test('selected shards partition the same files as the serial run exactly once', async () => {
+  const dir = await mkdtemp(join(tmpdir(), 'openclaude-runner-selection-'))
+  try {
+    const marker = join(dir, 'completed')
+    const fixtures = Array.from({ length: 8 }, (_, index) => join(dir, `fixture-${index + 1}.test.ts`))
+    for (const [index, fixture] of fixtures.entries()) {
+      await writeFile(fixture, `
+        import { test } from 'bun:test'
+        import { appendFileSync } from 'node:fs'
+        test('fixture ${index + 1}', () => appendFileSync(process.env.OC_TEST_MARKER!, '${index + 1}\\n'))
+      `)
+    }
+    const run = async (shardIndex?: number) => {
+      await writeFile(marker, '')
+      const child = Bun.spawn([
+        process.execPath, runner, '--shards=8',
+        ...(shardIndex === undefined ? [] : [`--shard-index=${shardIndex}`]),
+        ...fixtures,
+      ], { stdout: 'pipe', stderr: 'pipe', env: { ...process.env, OC_TEST_MARKER: marker } })
+      const [exitCode, stdout, stderr] = await Promise.all([
+        child.exited, new Response(child.stdout).text(), new Response(child.stderr).text(),
+      ])
+      expect(exitCode, stdout + stderr).toBe(0)
+      expect(stderr).toContain(`Completed ${shardIndex === undefined ? '8/8' : '1/1'} test shards.`)
+      const executed = (await readFile(marker, 'utf8')).trim().split('\n').sort()
+      return { executed, stderr }
+    }
+    const serial = await run()
+    expect(serial.executed).toEqual(['1', '2', '3', '4', '5', '6', '7', '8'])
+    const selected: string[] = []
+    for (let shardIndex = 1; shardIndex <= 8; shardIndex++) {
+      const result = await run(shardIndex)
+      expect(result.stderr.match(/Starting test shard \d+\/8/g)).toEqual([`Starting test shard ${shardIndex}/8`])
+      const serialShard = serial.stderr.split(`Starting test shard ${shardIndex}/8`)[1]!.split('Finished test shard')[0]!
+      const assigned = [...serialShard.matchAll(/fixture-(\d+)\.test\.ts:/g)].map(match => match[1]!).sort()
+      expect(result.executed).toEqual(assigned)
+      selected.push(...result.executed)
+    }
+    expect(selected.sort()).toEqual(serial.executed)
+  } finally {
+    await rm(dir, { recursive: true, force: true })
+  }
+}, 15_000)
+
+for (const options of [
+  ['--shard-index=0'],
+  ['--shard-index=-1'],
+  ['--shard-index=3'],
+  ['--shard-index=1.5'],
+  ['--shard-index=invalid'],
+  ['--shard-index=9007199254740992'],
+  ['--shard-index='],
+  ['--shard-index'],
+  ['--shard-index', '1'],
+  ['--shard-index=1', '--shard-index=2'],
+]) {
+  test(`test runner rejects invalid selection ${options.join(' ')}`, async () => {
+    const child = Bun.spawn([process.execPath, runner, '--shards=2', ...options], {
+      stdout: 'pipe', stderr: 'pipe',
+    })
+    const [exitCode, stdout, stderr] = await Promise.all([
+      child.exited, new Response(child.stdout).text(), new Response(child.stderr).text(),
+    ])
+    expect(exitCode, stdout + stderr).not.toBe(0)
+    expect(stderr).toContain('Use one --shard-index=')
+    expect(stderr).not.toContain('Starting test shard')
+  })
+}
+
+for (const scenario of ['assertion', 'exit'] as const) {
+  test(`selected shard detects ${scenario} failure`, async () => {
+    const dir = await mkdtemp(join(tmpdir(), 'openclaude-runner-selected-failure-'))
+    try {
+      const fixtures = [join(dir, 'first.test.ts'), join(dir, 'later.test.ts')]
+      for (const fixture of fixtures) {
+        await writeFile(fixture, `
+          import { test, expect } from 'bun:test'
+          test('failure', () => { ${scenario === 'exit' ? 'process.exit(0)' : 'expect(1).toBe(2)'} })
+        `)
+      }
+      const child = Bun.spawn([process.execPath, runner, '--shards=2', '--shard-index=2', ...fixtures], {
+        stdout: 'pipe', stderr: 'pipe',
+      })
+      const [exitCode, stdout, stderr] = await Promise.all([
+        child.exited, new Response(child.stdout).text(), new Response(child.stderr).text(),
+      ])
+      expect(exitCode, stdout + stderr).toBe(1)
+      expect(stderr.match(/Starting test shard \d+\/2/g)).toEqual(['Starting test shard 2/2'])
+      expect(stderr).toContain(`Completed ${scenario === 'exit' ? 0 : 1}/1 test shards.`)
+      if (scenario === 'exit') expect(stderr).toContain('without a complete JUnit report')
+    } finally {
+      await rm(dir, { recursive: true, force: true })
+    }
+  })
+}
+
 for (const signal of [undefined, 'SIGINT', 'SIGTERM'] as const) {
   test.skipIf(process.platform === 'win32')(`test runner kills a stuck shard and its child (${signal ?? 'deadline'})`, async () => {
     const dir = await mkdtemp(join(tmpdir(), 'openclaude-runner-hang-'))
@@ -135,7 +231,8 @@ for (const signal of [undefined, 'SIGINT', 'SIGTERM'] as const) {
         }, 200)
       `)
       const shardTimeout = signal ? 4000 : 1500
-      child = Bun.spawn([process.execPath, runner, `--shard-timeout-ms=${shardTimeout}`, fixture], {
+      child = Bun.spawn([process.execPath, runner, `--shard-timeout-ms=${shardTimeout}`,
+        ...(signal ? [] : ['--shards=2', '--shard-index=1']), fixture], {
         stdout: 'pipe', stderr: 'pipe', timeout: 6000, killSignal: 'SIGKILL',
       })
       const output = Promise.all([new Response(child.stdout).text(), new Response(child.stderr).text()])
@@ -152,7 +249,7 @@ for (const signal of [undefined, 'SIGINT', 'SIGTERM'] as const) {
       const [, stderr] = await output
       expect(stderr).toContain('last diagnostic before hang')
       expect(exitCode, stderr).toBe(signal === 'SIGINT' ? 130 : signal === 'SIGTERM' ? 143 : 124)
-      if (!signal) expect(stderr).toContain('Test shard 1/1 timed out after 1500ms.')
+      if (!signal) expect(stderr).toContain('Test shard 1/2 timed out after 1500ms.')
       // An orphaned compiler must not survive a timeout or cancellation.
       for (let attempt = 0; attempt < 100; attempt++) {
         try { process.kill(descendant!, 0) } catch { break }

scripts/run-tests.ts

@@ -6,23 +6,29 @@ import { join } from 'node:path'
 const args = process.argv.slice(2)
 const shardOptions = args.filter(arg => arg.startsWith('--shards='))
 const shards = shardOptions.length ? Number(shardOptions[0]!.slice('--shards='.length)) : 1
+const shardIndexOptions = args.filter(arg => arg === '--shard-index' || arg.startsWith('--shard-index='))
+const shardIndex = shardIndexOptions.length ? Number(shardIndexOptions[0]!.slice('--shard-index='.length)) : undefined
 const timeoutOptions = args.filter(arg => arg.startsWith('--shard-timeout-ms='))
 const timeoutMs = timeoutOptions.length ? Number(timeoutOptions[0]!.split('=')[1]) : 600_000
 if (shardOptions.length > 1 || !Number.isSafeInteger(shards) || shards < 1 ||
     args.some(arg => arg === '--shard' || arg.startsWith('--shard='))) {
   throw new Error('Use one --shards=<positive integer> option; native --shard is not supported.')
 }
+if (shardIndexOptions.length > 1 || (shardIndex !== undefined &&
+    (!Number.isSafeInteger(shardIndex) || shardIndex < 1 || shardIndex > shards))) {
+  throw new Error('Use one --shard-index=<integer from 1 to the shard count> option.')
+}
 if (timeoutOptions.length > 1 || !Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 2_147_483_647) {
   throw new Error('Use one --shard-timeout-ms=<positive integer up to 2147483647> option.')
 }
-const testArgs = args.filter(arg => !arg.startsWith('--shards=') && !arg.startsWith('--shard-timeout-ms='))
+const testArgs = args.filter(arg => !arg.startsWith('--shards=') && !arg.startsWith('--shard-index=') && !arg.startsWith('--shard-timeout-ms='))
 // Bun can exit zero without finishing its suite if a test calls process.exit(0).
 // Its JUnit report is written only when the runner finishes; require a fresh one.
 const reportDir = await mkdtemp(join(tmpdir(), 'openclaude-test-report-'))
 let exitCode = 0
 let completed = 0
 try {
-  for (let shard = 1; shard <= shards; shard++) {
+  for (let shard = shardIndex ?? 1; shard <= (shardIndex ?? shards); shard++) {
     const started = performance.now()
     console.error(`Starting test shard ${shard}/${shards} (deadline ${timeoutMs}ms).`)
     const reportPath = join(reportDir, `results-${shard}.xml`)
@@ -138,7 +144,7 @@ try {
       }
     }
   }
-  if (shards > 1) console.error(`Completed ${completed}/${shards} test shards.`)
+  if (shards > 1) console.error(`Completed ${completed}/${shardIndex === undefined ? shards : 1} test shards.`)
 } finally {
   await rm(reportDir, { recursive: true, force: true })
 }