fix(deployments): rebuild when a build secret value changes

Docker ignores secret values for the build cache, so the COOLIFY_BUILD_SECRETS_HASH secret mount never invalidated it. Pass the hash as a build arg and declare it in each stage instead. Parse Dockerfile instructions so ARG lines and secret mounts are not added inside heredoc bodies or continuation lines. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 13:10:37 -04:00 · 2026-10-26 12:56:31 +01:00
parent 25e2eae058
commit a3843730f8
3 changed files with 226 additions and 32 deletions
@@ -5270,8 +5270,6 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
            return '';
        }

-        $secrets_hash = $this->generate_secrets_hash($variables);
-
        // Get database env vars to check for multiline flag
        $env_vars = $this->pull_request_id === 0
            ? $this->application->environment_variables()->where('is_buildtime', true)->get()
@@ -5289,10 +5287,7 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
            ];
        });

-        $env_flags = generateDockerEnvFlags($vars_array);
-        $env_flags .= ' -e '.escapeshellarg("COOLIFY_BUILD_SECRETS_HASH={$secrets_hash}");
-
-        return $env_flags;
+        return generateDockerEnvFlags($vars_array);
    }

    private function generate_build_secrets(Collection $variables)
@@ -5311,7 +5306,8 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
            })
            ->implode(' ');

-        $this->build_secrets .= ' --secret id=COOLIFY_BUILD_SECRETS_HASH,env=COOLIFY_BUILD_SECRETS_HASH';
+        // Docker ignores secret values for the build cache, so the hash goes in as a build arg
+        $this->build_secrets .= ' --build-arg '.escapeshellarg('COOLIFY_BUILD_SECRETS_HASH='.$this->generate_secrets_hash($variables));
    }

    private function generate_secrets_hash($variables)
@@ -5344,18 +5340,66 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
        return hash_hmac('sha256', $secrets_string, $this->secrets_hash_key);
    }

+    /**
+     * Returns the last line of each FROM instruction, so stage ARGs can go directly after it.
+     */
    protected function findFromInstructionLines($dockerfile): array
    {
-        $fromLines = [];
+        return collect($this->dockerfileInstructions($dockerfile))
+            ->where('keyword', 'FROM')
+            ->pluck('end')
+            ->all();
+    }
+
+    /**
+     * Splits a Dockerfile into instructions with their first and last line. Comments, continuation
+     * lines, and heredoc bodies are part of an instruction, so text such as "from app import x"
+     * in a heredoc is not read as an instruction.
+     *
+     * @return list<array{keyword: string, start: int, end: int}>
+     */
+    protected function dockerfileInstructions(iterable $dockerfile): array
+    {
+        $escape = '\\';
+        $instructions = [];
+        $continues = false;
+        $heredocTerminators = [];
+
        foreach ($dockerfile as $index => $line) {
-            $trimmedLine = trim($line);
-            // Check if line starts with FROM (case-insensitive)
-            if (preg_match('/^FROM\s+/i', $trimmedLine)) {
-                $fromLines[] = $index;
+            $trimmed = trim($line);
+
+            if ($heredocTerminators !== []) {
+                $instructions[array_key_last($instructions)]['end'] = $index;
+                if ($trimmed === $heredocTerminators[0]) {
+                    array_shift($heredocTerminators);
+                }
+
+                continue;
+            }
+
+            if ($trimmed === '' || str_starts_with($trimmed, '#')) {
+                if ($instructions === [] && preg_match('/^#\s*escape\s*=\s*([\\\\`])\s*$/i', $trimmed, $matches) === 1) {
+                    $escape = $matches[1];
+                }
+
+                continue;
+            }
+
+            if ($continues) {
+                $instructions[array_key_last($instructions)]['end'] = $index;
+            } else {
+                $instructions[] = ['keyword' => strtoupper(strtok($trimmed, " \t")), 'start' => $index, 'end' => $index];
+            }
+            $continues = str_ends_with($trimmed, $escape);
+
+            $keyword = $instructions[array_key_last($instructions)]['keyword'];
+            if (in_array($keyword, ['RUN', 'COPY', 'ADD'], true)
+                && preg_match_all('/(?<!<)<<-?(["\']?)([A-Za-z_][A-Za-z0-9_]*)\1/', $line, $matches) > 0) {
+                $heredocTerminators = $matches[2];
            }
        }

-        return $fromLines;
+        return $instructions;
    }

    private function add_build_env_variables_to_dockerfile()
@@ -5516,23 +5560,27 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
            return "--mount=type=secret,id={$key},env={$key}";
        })->implode(' ');

-        // Add mount for the secrets hash to ensure cache invalidation
-        $mountStrings .= ' --mount=type=secret,id=COOLIFY_BUILD_SECRETS_HASH,env=COOLIFY_BUILD_SECRETS_HASH';
+        $instructions = collect($this->dockerfileInstructions($dockerfile));
+        $runLines = $instructions->where('keyword', 'RUN')->pluck('start')->all();
+        $fromEndLines = $instructions->where('keyword', 'FROM')->pluck('end')->all();

        $modified = false;
-        $dockerfile = $dockerfile->map(function ($line) use ($mountStrings, &$modified) {
-            $trimmed = ltrim($line);
-
-            // Skip lines that already have secret mounts or are not RUN commands
-            if (str_contains($line, '--mount=type=secret') || ! str_starts_with($trimmed, 'RUN')) {
-                return $line;
+        $dockerfile = $dockerfile->flatMap(function ($line, $index) use ($mountStrings, $runLines, $fromEndLines, &$modified) {
+            // Add mount strings to RUN commands that do not have secret mounts yet
+            if (in_array($index, $runLines, true) && ! str_contains($line, '--mount=type=secret')) {
+                $originalCommand = trim(substr(ltrim($line), 3));
+                $line = "RUN {$mountStrings} {$originalCommand}";
+                $modified = true;
            }

-            // Add mount strings to RUN command
-            $originalCommand = trim(substr($trimmed, 3));
-            $modified = true;
+            // A RUN step reads every ARG of its stage, so a changed hash rebuilds it
+            if (in_array($index, $fromEndLines, true)) {
+                $modified = true;

-            return "RUN {$mountStrings} {$originalCommand}";
+                return [$line, 'ARG COOLIFY_BUILD_SECRETS_HASH'];
+            }
+
+            return [$line];
        });

        if ($modified) {
@@ -5617,12 +5665,7 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");

            $dockerfile_lines = collect(str($dockerfileContent)->trim()->explode("\n"));

-            $fromIndices = [];
-            $dockerfile_lines->each(function ($line, $index) use (&$fromIndices) {
-                if (str($line)->trim()->startsWith('FROM')) {
-                    $fromIndices[] = $index;
-                }
-            });
+            $fromIndices = $this->findFromInstructionLines($dockerfile_lines);

            if (empty($fromIndices)) {
                $this->application_deployment_queue->addLogEntry("No FROM instruction found in Dockerfile for service {$serviceName}, skipping.");
@@ -5795,6 +5838,7 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
                'environment' => $key,
            ];
        }
+        $secretsHash = $this->generate_secrets_hash($variables);

        $services = data_get($composeFile, 'services', []);
        foreach ($services as $serviceName => &$service) {
@@ -5812,6 +5856,18 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
                        $service['build']['secrets'][] = $key;
                    }
                }
+
+                // Docker ignores secret values for the build cache, so the hash goes in as a build arg
+                $args = $service['build']['args'] ?? [];
+                if ($args instanceof Collection) {
+                    $args = $args->toArray();
+                }
+                if ($args !== [] && array_is_list($args)) {
+                    $args[] = "COOLIFY_BUILD_SECRETS_HASH={$secretsHash}";
+                } else {
+                    $args['COOLIFY_BUILD_SECRETS_HASH'] = $secretsHash;
+                }
+                $service['build']['args'] = $args;
            }
        }

@@ -1320,7 +1320,6 @@ it('keeps invalid build-time names out of the helper container so restarts with

    expect($envFlags)
        ->toContain("-e 'SAFE_KEY=safe-build-value'")
-        ->toContain('COOLIFY_BUILD_SECRETS_HASH=')
        ->not->toContain('MY-VAR')
        ->not->toContain('invalid-build-value');
 });
@@ -1939,3 +1938,94 @@ it('includes migrated null Compose overrides in production and preview runtime f

    expect(invokeDeploymentJobMethod($job, $reflection, 'generate_runtime_environment_variables')->contains('LEGACY='.escapeComposeEnvFileValue('')))->toBeTrue();
 })->with([0, 42]);
+
+it('passes the build secrets hash as a build arg so a changed secret value invalidates the Docker cache', function () {
+    [$application, $server] = makeDeploymentControlVarFixture();
+    [$job, $reflection] = makeControlVarFilteringJob($application, $server, ['dockerSecretsSupported' => true]);
+
+    invokeDeploymentJobMethod($job, $reflection, 'generate_build_secrets', collect(['NEXT_PUBLIC_API_URL' => 'https://old.example.com']));
+    $oldFlags = readDeploymentJobProperty($job, $reflection, 'build_secrets');
+
+    invokeDeploymentJobMethod($job, $reflection, 'generate_build_secrets', collect(['NEXT_PUBLIC_API_URL' => 'https://new.example.com']));
+    $newFlags = readDeploymentJobProperty($job, $reflection, 'build_secrets');
+
+    $hashArg = fn (string $flags): ?string => preg_match("/--build-arg 'COOLIFY_BUILD_SECRETS_HASH=([a-f0-9]{64})'/", $flags, $matches) === 1 ? $matches[1] : null;
+
+    expect($hashArg($oldFlags))->not->toBeNull()
+        ->and($hashArg($newFlags))->not->toBeNull()
+        ->and($hashArg($newFlags))->not->toBe($hashArg($oldFlags))
+        ->and($newFlags)->toContain("--secret 'id=NEXT_PUBLIC_API_URL,env=NEXT_PUBLIC_API_URL'");
+});
+
+it('declares the build secrets hash ARG in every stage when build secrets are used', function () {
+    [$application, $server] = makeDeploymentControlVarFixture();
+    [$job, $reflection] = makeControlVarFilteringJob($application, $server, [
+        'dockerSecretsSupported' => true,
+        'env_args' => collect(['NEXT_PUBLIC_API_URL' => 'https://new.example.com']),
+        'build_secrets' => '--secret id=NEXT_PUBLIC_API_URL,env=NEXT_PUBLIC_API_URL',
+        'saved_outputs' => collect([
+            'dockerfile_content' => "FROM node:22 AS build\nRUN npm run build\nFROM nginx:alpine\nCOPY --from=build /app/dist /usr/share/nginx/html",
+        ]),
+    ]);
+
+    invokeDeploymentJobMethod($job, $reflection, 'modify_dockerfile_for_secrets', '/artifacts/test-app/Dockerfile');
+
+    expect(explode("\n", $job->writtenDockerfile))->toBe([
+        '# syntax=docker/dockerfile:1',
+        'FROM node:22 AS build',
+        'ARG COOLIFY_BUILD_SECRETS_HASH',
+        'RUN --mount=type=secret,id=NEXT_PUBLIC_API_URL,env=NEXT_PUBLIC_API_URL npm run build',
+        'FROM nginx:alpine',
+        'ARG COOLIFY_BUILD_SECRETS_HASH',
+        'COPY --from=build /app/dist /usr/share/nginx/html',
+    ]);
+});
+
+it('passes the build secrets hash as a Compose build arg when build secrets are used', function () {
+    [$application, $server] = makeDeploymentControlVarFixture(['build_pack' => 'dockercompose']);
+    [$job, $reflection] = makeControlVarFilteringJob($application, $server, [
+        'dockerSecretsSupported' => true,
+        'env_args' => collect(['NEXT_PUBLIC_API_URL' => 'https://new.example.com']),
+    ]);
+
+    $composeFile = invokeDeploymentJobMethod($job, $reflection, 'add_build_secrets_to_compose', [
+        'services' => [
+            'web' => ['build' => '.'],
+            'api' => ['build' => ['context' => './api', 'args' => ['NODE_ENV' => 'production']]],
+            'worker' => ['build' => ['context' => './worker', 'args' => ['NODE_ENV=production']]],
+            'db' => ['image' => 'postgres:17'],
+        ],
+    ]);
+
+    $hash = data_get($composeFile, 'services.web.build.args.COOLIFY_BUILD_SECRETS_HASH');
+
+    expect($hash)->toMatch('/^[a-f0-9]{64}$/')
+        ->and(data_get($composeFile, 'services.api.build.args'))->toBe(['NODE_ENV' => 'production', 'COOLIFY_BUILD_SECRETS_HASH' => $hash])
+        ->and(data_get($composeFile, 'services.worker.build.args'))->toBe(['NODE_ENV=production', "COOLIFY_BUILD_SECRETS_HASH={$hash}"])
+        ->and(data_get($composeFile, 'services.db'))->toBe(['image' => 'postgres:17']);
+});
+
+it('keeps heredoc bodies unchanged when it adds build secrets to a Dockerfile', function () {
+    [$application, $server] = makeDeploymentControlVarFixture();
+    [$job, $reflection] = makeControlVarFilteringJob($application, $server, [
+        'dockerSecretsSupported' => true,
+        'env_args' => collect(['API_URL' => 'https://new.example.com']),
+        'build_secrets' => '--secret id=API_URL,env=API_URL',
+        'saved_outputs' => collect([
+            'dockerfile_content' => "FROM python:3.13\nRUN <<EOF\nfrom app import build\nRUN-like line\nEOF\nRUN python -c 'print(1)'",
+        ]),
+    ]);
+
+    invokeDeploymentJobMethod($job, $reflection, 'modify_dockerfile_for_secrets', '/artifacts/test-app/Dockerfile');
+
+    expect(explode("\n", $job->writtenDockerfile))->toBe([
+        '# syntax=docker/dockerfile:1',
+        'FROM python:3.13',
+        'ARG COOLIFY_BUILD_SECRETS_HASH',
+        'RUN --mount=type=secret,id=API_URL,env=API_URL <<EOF',
+        'from app import build',
+        'RUN-like line',
+        'EOF',
+        "RUN --mount=type=secret,id=API_URL,env=API_URL python -c 'print(1)'",
+    ]);
+});
@@ -80,6 +80,54 @@ it('returns empty array when no FROM instructions found', function () {
    expect($result)->toBe([]);
 });

+it('ignores FROM text inside heredoc bodies', function () {
+    $job = Mockery::mock(ApplicationDeploymentJob::class)->makePartial();
+
+    $dockerfile = collect([
+        'FROM python:3.13 AS build',
+        'RUN <<EOF',
+        'from app import create_app',
+        'EOF',
+        "COPY <<-'SQL' /init.sql",
+        "\tSELECT *",
+        "\tFROM users;",
+        "\tSQL",
+        'RUN cat <<<"from here-string"',
+        'FROM nginx:alpine',
+    ]);
+
+    expect($job->findFromInstructionLines($dockerfile))->toBe([0, 9]);
+});
+
+it('returns the last line of a FROM instruction that continues on the next line', function () {
+    $job = Mockery::mock(ApplicationDeploymentJob::class)->makePartial();
+
+    $dockerfile = collect([
+        'FROM --platform=linux/amd64 \\',
+        '    # base image',
+        '    node:22',
+        'RUN echo \\',
+        '    FROM not-an-instruction',
+        'FROM nginx:alpine',
+    ]);
+
+    expect($job->findFromInstructionLines($dockerfile))->toBe([2, 5]);
+});
+
+it('uses the escape parser directive for continuation lines', function () {
+    $job = Mockery::mock(ApplicationDeploymentJob::class)->makePartial();
+
+    $dockerfile = collect([
+        '# escape=`',
+        'FROM mcr.microsoft.com/windows/servercore:ltsc2022',
+        'RUN dir `',
+        '    FROM c:\\',
+        'FROM nginx:alpine',
+    ]);
+
+    expect($job->findFromInstructionLines($dockerfile))->toBe([1, 4]);
+});
+
 it('inserts ARGs after FROM in simple dockerfile', function () {
    $dockerfile = collect([
        'FROM node:16',