fix(storage): require deploy permission for external host writes

Require deploy or root token access when creating managed mounts outside
resource directories or changing their file contents through the API.

Synchronize KeyDB and Dragonfly passwords with local and shared
REDIS_PASSWORD values while preserving secret manager resolution.

tests/Feature/ApplicationComposeInjectionValidationTest.php

@@ -356,3 +356,24 @@ test('the Compose deployment validates the repository file before any command us
         ->and($loadPosition)->toBeLessThan(strpos($body, 'base64 -d'))
         ->and($body)->toContain('"stat -c \'%F\' ".escapeshellarg($realPathInGit)');
 });
+
+test('loading a Compose file with a variable service network saves it', function () {
+    $compose = "services:\n  web:\n    image: nginx\n    networks:\n      - \${NET:-proxy}\nnetworks:\n  proxy:\n    external: true\n";
+    fakeRepositoryCompose($compose);
+
+    $this->application->loadComposeFile();
+
+    expect($this->application->fresh()->docker_compose_raw)->toBe(trim($compose));
+});
+
+test('the deployment log names the service and network that failed validation', function () {
+    fakeRepositoryCompose("services:\n  web:\n    image: nginx\n    networks:\n      - '\$(id)'\n");
+    $logEntries = [];
+    $job = composeDeploymentJob($this->application, 0, $logEntries);
+
+    expect(fn () => (new ReflectionMethod(ApplicationDeploymentJob::class, 'loadComposeFileForDeployment'))->invoke($job))
+        ->toThrow(DeploymentException::class);
+
+    expect(collect($logEntries)->pluck(0)->implode("\n"))
+        ->toContain('Invalid Docker Compose service network "$(id)" in service web.');
+});

tests/Feature/GithubRunnerFlowTest.php

@@ -270,6 +270,154 @@ describe('workflow_job webhook', function () {
             ->and($execution->started_at)->not->toBeNull();
     });
 
+    it('starts a new runner for the job whose runner another job took', function () {
+        $stolen = runnerTestExecution($this->githubApp, [
+            'trigger_workflow_job_id' => 5001,
+            'workflow_job_id' => 5001,
+            'job_name' => 'build',
+            'status' => GithubRunnerStatus::Idle,
+            'server_id' => $this->server->id,
+            'runner_name' => 'coolify-runner-a',
+            'provision_attempts' => 1,
+        ]);
+
+        foreach ([1, 2] as $delivery) {
+            sendWorkflowJobWebhook($this, $this->githubApp, 'runner-secret', 'in_progress', [
+                'id' => 6001,
+                'name' => 'pr-test',
+                'runner_name' => 'coolify-runner-a',
+            ])->assertOk();
+        }
+
+        $stolen->refresh();
+        $replacement = GithubRunnerExecution::query()->where('trigger_workflow_job_id', 5001)->sole();
+        expect($stolen->status)->toBe(GithubRunnerStatus::Running)
+            ->and($stolen->trigger_workflow_job_id)->toBe(6001)
+            ->and($stolen->workflow_job_id)->toBe(6001)
+            ->and($replacement->id)->not->toBe($stolen->id)
+            ->and($replacement->status)->toBe(GithubRunnerStatus::Queued)
+            ->and($replacement->job_name)->toBe('build')
+            ->and($replacement->labels)->toBe(['self-hosted', 'coolify'])
+            ->and($replacement->provision_attempts)->toBe(1);
+        Queue::assertPushed(ProvisionGithubRunnerJob::class, 1);
+        Queue::assertPushed(ProvisionGithubRunnerJob::class, fn ($job) => $job->executionId === $replacement->id);
+
+        sendWorkflowJobWebhook($this, $this->githubApp, 'runner-secret', 'completed', [
+            'id' => 6001,
+            'runner_name' => 'coolify-runner-a',
+            'conclusion' => 'success',
+        ])->assertOk();
+
+        expect($stolen->fresh()->status)->toBe(GithubRunnerStatus::Completed)
+            ->and($replacement->fresh()->status)->toBe(GithubRunnerStatus::Queued);
+    });
+
+    it('starts a new runner when the completed hook of the other job arrives before its in_progress hook', function () {
+        $stolen = runnerTestExecution($this->githubApp, [
+            'trigger_workflow_job_id' => 5001,
+            'status' => GithubRunnerStatus::Idle,
+            'server_id' => $this->server->id,
+            'runner_name' => 'coolify-runner-a',
+            'provision_attempts' => 1,
+        ]);
+
+        sendWorkflowJobWebhook($this, $this->githubApp, 'runner-secret', 'completed', [
+            'id' => 6001,
+            'runner_name' => 'coolify-runner-a',
+            'conclusion' => 'success',
+        ])->assertOk();
+
+        expect($stolen->fresh()->status)->toBe(GithubRunnerStatus::Completed)
+            ->and($stolen->fresh()->trigger_workflow_job_id)->toBe(6001)
+            ->and(GithubRunnerExecution::query()->where('trigger_workflow_job_id', 5001)->sole()->status)->toBe(GithubRunnerStatus::Queued);
+        Queue::assertPushed(ProvisionGithubRunnerJob::class, 1);
+    });
+
+    it('replaces a stale execution of the job that took the runner', function () {
+        $stale = runnerTestExecution($this->githubApp, ['trigger_workflow_job_id' => 6001, 'status' => GithubRunnerStatus::TimedOut]);
+        $stolen = runnerTestExecution($this->githubApp, [
+            'trigger_workflow_job_id' => 5001,
+            'status' => GithubRunnerStatus::Idle,
+            'server_id' => $this->server->id,
+            'runner_name' => 'coolify-runner-a',
+            'provision_attempts' => 1,
+        ]);
+
+        sendWorkflowJobWebhook($this, $this->githubApp, 'runner-secret', 'in_progress', [
+            'id' => 6001,
+            'runner_name' => 'coolify-runner-a',
+        ])->assertOk();
+
+        expect($stale->fresh())->toBeNull()
+            ->and($stolen->fresh()->trigger_workflow_job_id)->toBe(6001)
+            ->and(GithubRunnerExecution::query()->where('trigger_workflow_job_id', 5001)->sole()->status)->toBe(GithubRunnerStatus::Queued);
+        Queue::assertPushed(ProvisionGithubRunnerJob::class, 1);
+    });
+
+    it('does not start a new runner when the runner took another job with its own pending runner', function () {
+        $first = runnerTestExecution($this->githubApp, [
+            'trigger_workflow_job_id' => 5001,
+            'status' => GithubRunnerStatus::Idle,
+            'server_id' => $this->server->id,
+            'runner_name' => 'coolify-runner-a',
+        ]);
+        $second = runnerTestExecution($this->githubApp, ['trigger_workflow_job_id' => 6001]);
+
+        sendWorkflowJobWebhook($this, $this->githubApp, 'runner-secret', 'in_progress', [
+            'id' => 6001,
+            'runner_name' => 'coolify-runner-a',
+        ])->assertOk();
+
+        expect(GithubRunnerExecution::count())->toBe(2)
+            ->and($first->fresh()->status)->toBe(GithubRunnerStatus::Running)
+            ->and($first->fresh()->workflow_job_id)->toBe(6001)
+            ->and($second->fresh()->status)->toBe(GithubRunnerStatus::Queued);
+        Queue::assertNotPushed(ProvisionGithubRunnerJob::class);
+    });
+
+    it('does not start a new runner for a job whose runner was taken', function (Closure $setup, array $attributes) {
+        $setup();
+        $stolen = runnerTestExecution($this->githubApp, [
+            'trigger_workflow_job_id' => 5001,
+            'status' => GithubRunnerStatus::Idle,
+            'server_id' => $this->server->id,
+            'runner_name' => 'coolify-runner-a',
+            'provision_attempts' => 1,
+            ...$attributes,
+        ]);
+
+        sendWorkflowJobWebhook($this, $this->githubApp, 'runner-secret', 'in_progress', [
+            'id' => 6001,
+            'runner_name' => 'coolify-runner-a',
+        ])->assertOk();
+
+        expect(GithubRunnerExecution::count())->toBe(1)
+            ->and($stolen->fresh()->status)->toBe(GithubRunnerStatus::Running);
+        Queue::assertNotPushed(ProvisionGithubRunnerJob::class);
+    })->with([
+        'runner limit of the job reached' => [fn () => null, ['provision_attempts' => ProvisionGithubRunnerJob::MAX_ATTEMPTS]],
+        'configuration disabled' => [fn () => GithubRunnerConfig::query()->update(['is_enabled' => false]), []],
+        'pull request job no longer allowed' => [fn () => null, ['is_pull_request' => true]],
+    ]);
+
+    it('marks the runner running without a new runner when it took the job it was started for', function () {
+        $execution = runnerTestExecution($this->githubApp, [
+            'trigger_workflow_job_id' => 5001,
+            'status' => GithubRunnerStatus::Idle,
+            'server_id' => $this->server->id,
+            'runner_name' => 'coolify-runner-a',
+        ]);
+
+        sendWorkflowJobWebhook($this, $this->githubApp, 'runner-secret', 'in_progress', ['runner_name' => 'coolify-runner-a'])->assertOk();
+
+        $execution->refresh();
+        expect(GithubRunnerExecution::count())->toBe(1)
+            ->and($execution->status)->toBe(GithubRunnerStatus::Running)
+            ->and($execution->trigger_workflow_job_id)->toBe(5001)
+            ->and($execution->workflow_job_id)->toBe(5001);
+        Queue::assertNotPushed(ProvisionGithubRunnerJob::class);
+    });
+
     it('completes the runner and queues its cleanup', function () {
         $execution = runnerTestExecution($this->githubApp, [
             'status' => GithubRunnerStatus::Running,

tests/Feature/KeydbDragonflyLegacyPasswordQuotingTest.php

@@ -8,6 +8,7 @@ use App\Models\IntegrationToken;
 use App\Models\PrivateKey;
 use App\Models\Project;
 use App\Models\Server;
+use App\Models\SharedEnvironmentVariable;
 use App\Models\StandaloneDocker;
 use App\Models\Team;
 use App\Services\DatabaseStartCommandExecutor;
@@ -51,7 +52,7 @@ beforeEach(function () {
 });
 
 /**
- * @return array{command: string, healthcheck: list<string>}
+ * @return array{command: string, healthcheck: list<string>, environment: list<string>}
  */
 function legacyPasswordQuotingStartedService(object $executor, string $uuid): array
 {
@@ -59,7 +60,7 @@ function legacyPasswordQuotingStartedService(object $executor, string $uuid): ar
         if (preg_match("#^echo '([^']+)' \| base64 -d \| tee \S+/docker-compose\.yml#", $command, $matches)) {
             $service = Yaml::parse(base64_decode($matches[1]))['services'][$uuid];
 
-            return ['command' => $service['command'], 'healthcheck' => $service['healthcheck']['test']];
+            return ['command' => $service['command'], 'healthcheck' => $service['healthcheck']['test'], 'environment' => $service['environment']];
         }
     }
 
@@ -195,3 +196,71 @@ it('quotes shell control characters in the password of a database created after
     'space' => 'a b',
     'pipe' => 'x|y',
 ]);
+
+it('runs with the stored password and syncs a plain REDIS_PASSWORD variable to it', function (string $create, string $start, string $passwordColumn, string $commandFormat) {
+    $database = $create($this->environment->id, $this->destination, [$passwordColumn => 'storedpass']);
+    $env = $database->runtime_environment_variables()->create(['key' => 'REDIS_PASSWORD', 'value' => 'envpass']);
+
+    $start::run($database->fresh(), new Activity);
+
+    $service = legacyPasswordQuotingStartedService($this->executor, $database->uuid);
+    expect($service['command'])->toBe(sprintf($commandFormat, "'storedpass'"))
+        ->and($service['healthcheck'][3])->toBe('storedpass')
+        ->and($service['environment'])->toContain('REDIS_PASSWORD=storedpass')
+        ->and($env->fresh()->value)->toBe('storedpass')
+        ->and($database->fresh()->{$passwordColumn})->toBe('storedpass');
+})->with('legacy-password-quoting-engines');
+
+it('keeps the exact v4.3.23 command of a legacy database with a different plain REDIS_PASSWORD variable', function (string $create, string $start, string $passwordColumn, string $commandFormat) {
+    $database = $create($this->environment->id, $this->destination, [$passwordColumn => 'stored;pass']);
+    DB::table($database->getTable())->where('id', $database->id)->update(['legacy_password_quoting' => true]);
+    $database->runtime_environment_variables()->create(['key' => 'REDIS_PASSWORD', 'value' => 'envpass']);
+
+    $start::run($database->fresh(), new Activity);
+
+    $service = legacyPasswordQuotingStartedService($this->executor, $database->uuid);
+    expect($service['command'])->toBe(sprintf($commandFormat, 'stored;pass'))
+        ->and($service['healthcheck'][3])->toBe('stored;pass')
+        ->and($database->fresh()->legacy_password_quoting)->toBeTrue();
+})->with('legacy-password-quoting-engines');
+
+it('runs with a remote secret REDIS_PASSWORD without storing it', function (string $create, string $start, string $passwordColumn, string $commandFormat) {
+    Http::fake(['https://api.doppler.com/*' => Http::response(['REDIS_PASSWORD' => 'secretpass'])]);
+    $token = IntegrationToken::query()->create([
+        'team_id' => $this->environment->project->team_id,
+        'provider' => 'doppler',
+        'name' => 'Doppler',
+        'token' => 'the-secret-token',
+        'capabilities' => ['secrets'],
+    ]);
+    $database = $create($this->environment->id, $this->destination, [$passwordColumn => 'storedpass']);
+    $database->secretManagerLink()->create(['integration_token_id' => $token->id]);
+    $env = $database->runtime_environment_variables()->create(['key' => 'REDIS_PASSWORD', 'value' => '{{vault.REDIS_PASSWORD}}']);
+
+    $start::run($database->fresh(), new Activity);
+
+    $service = legacyPasswordQuotingStartedService($this->executor, $database->uuid);
+    expect($service['command'])->toBe(sprintf($commandFormat, "'secretpass'"))
+        ->and($service['healthcheck'][3])->toBe('secretpass')
+        ->and($env->fresh()->value)->toBe('{{vault.REDIS_PASSWORD}}')
+        ->and($database->fresh()->{$passwordColumn})->toBe('storedpass');
+})->with('legacy-password-quoting-engines');
+
+it('copies a shared REDIS_PASSWORD variable into the stored password', function (string $create, string $start, string $passwordColumn, string $commandFormat) {
+    SharedEnvironmentVariable::query()->create([
+        'key' => 'REDIS_PASSWORD',
+        'value' => 'sharedpass',
+        'type' => 'team',
+        'team_id' => $this->environment->project->team_id,
+    ]);
+    $database = $create($this->environment->id, $this->destination, [$passwordColumn => 'storedpass']);
+    $env = $database->runtime_environment_variables()->create(['key' => 'REDIS_PASSWORD', 'value' => '{{team.REDIS_PASSWORD}}']);
+
+    $start::run($database->fresh(), new Activity);
+
+    $service = legacyPasswordQuotingStartedService($this->executor, $database->uuid);
+    expect($service['command'])->toBe(sprintf($commandFormat, "'sharedpass'"))
+        ->and($service['healthcheck'][3])->toBe('sharedpass')
+        ->and($env->fresh()->value)->toBe('{{team.REDIS_PASSWORD}}')
+        ->and($database->fresh()->{$passwordColumn})->toBe('sharedpass');
+})->with('legacy-password-quoting-engines');

tests/Feature/MultiServerApplicationTest.php

@@ -112,6 +112,13 @@ describe('registry image guard', function () {
         callMultiServerDeploymentJob($job, 'ensureRegistryImageForMultipleServers');
     })->throwsNoExceptions();
 
+    test('allows docker compose applications, which build on each server', function () {
+        $this->application->update(['build_pack' => 'dockercompose']);
+        $job = makeMultiServerDeploymentJob(['application' => $this->application->fresh()]);
+
+        callMultiServerDeploymentJob($job, 'ensureRegistryImageForMultipleServers');
+    })->throwsNoExceptions();
+
     test('allows preview deployments, which run only on the main server', function () {
         $job = makeMultiServerDeploymentJob(['application' => $this->application, 'pull_request_id' => 7]);
 

tests/Feature/StorageHostPathTest.php

@@ -1,5 +1,6 @@
 <?php
 
+use App\Jobs\ServerStorageSaveJob;
 use App\Livewire\Project\Service\FileStorage;
 use App\Livewire\Project\Service\Storage;
 use App\Models\Application;
@@ -70,7 +71,10 @@ function storageHostPathFakeServer(string $symlink = 'OK', string $exists = 'NOK
     }));
 }
 
-function storageHostPathToken(User $user): string
+/**
+ * @param  list<string>  $abilities
+ */
+function storageHostPathToken(User $user, array $abilities = ['*']): string
 {
     // A token request must not use the session user of the Livewire tests.
     auth()->forgetGuards();
@@ -78,7 +82,7 @@ function storageHostPathToken(User $user): string
     $token = $user->tokens()->create([
         'name' => 'host-path-token',
         'token' => hash('sha256', $plainTextToken),
-        'abilities' => ['*'],
+        'abilities' => $abilities,
         'team_id' => test()->team->id,
     ]);
 
@@ -363,3 +367,85 @@ test('relative host paths stay inside the resource directory', function () {
     Process::assertRan(fn ($process) => str_contains($process->command, 'readlink -f')
         && str_contains($process->command, "'{$workdir}' '{$workdir}/data/zz'"));
 });
+
+test('a token without deploy cannot create a mount outside the resource directory', function (string $resourceType, array $mount) {
+    [$url, $payload] = storageHostPathApiTarget($resourceType);
+
+    $this->withHeaders(['Authorization' => 'Bearer '.storageHostPathToken($this->admin, ['read', 'write'])])
+        ->postJson($url, [...$payload, 'type' => 'file', ...$mount])
+        ->assertForbidden()
+        ->assertJsonPath('message', 'Missing required permissions: deploy. A mount outside the resource directory needs a token with the deploy permission.');
+
+    expect(LocalFileVolume::query()->count())->toBe(0);
+    Bus::assertNotDispatched(ServerStorageSaveJob::class);
+    Process::assertNothingRan();
+})->with(['application', 'database', 'service'])->with([
+    'directory' => [['is_directory' => true, 'fs_path' => '/root/.ssh', 'mount_path' => '/data']],
+    'file with content' => [['fs_path' => '/root/.ssh/authorized_keys', 'mount_path' => '/etc/keys', 'content' => 'ssh-ed25519 AAAA']],
+    'empty file' => [['fs_path' => '/etc/zz/app.conf', 'mount_path' => '/etc/app.conf']],
+]);
+
+test('a deploy or root token creates a mount outside the resource directory', function (string $resourceType, array $abilities) {
+    [$url, $payload, $resource] = storageHostPathApiTarget($resourceType);
+    $token = storageHostPathToken($this->admin, $abilities);
+
+    $this->withHeaders(['Authorization' => 'Bearer '.$token])
+        ->postJson($url, [...$payload, 'type' => 'file', 'is_directory' => true, 'fs_path' => '/srv/zz', 'mount_path' => '/data'])
+        ->assertCreated();
+    $this->withHeaders(['Authorization' => 'Bearer '.$token])
+        ->postJson($url, [...$payload, 'type' => 'file', 'fs_path' => '/etc/zz/app.conf', 'mount_path' => '/etc/app.conf', 'content' => 'listen 8080;'])
+        ->assertCreated();
+
+    expect($resource->fileStorages()->orderBy('id')->pluck('fs_path')->all())->toBe(['/srv/zz', '/etc/zz/app.conf']);
+})->with(['application', 'database', 'service'])->with([
+    'deploy' => [['read', 'write', 'deploy']],
+    'root' => [['root']],
+]);
+
+test('a write token creates mounts inside the resource directory', function (string $resourceType) {
+    [$url, $payload, $resource] = storageHostPathApiTarget($resourceType);
+    $token = storageHostPathToken($this->admin, ['read', 'write']);
+    $workdir = $resource instanceof ServiceApplication ? $resource->service->workdir() : $resource->workdir();
+
+    $this->withHeaders(['Authorization' => 'Bearer '.$token])
+        ->postJson($url, [...$payload, 'type' => 'file', 'is_directory' => true, 'fs_path' => 'data/zz', 'mount_path' => '/data'])
+        ->assertCreated();
+    $this->withHeaders(['Authorization' => 'Bearer '.$token])
+        ->postJson($url, [...$payload, 'type' => 'file', 'fs_path' => $workdir.'/config/app.conf', 'mount_path' => '/etc/app.conf', 'content' => 'listen 8080;'])
+        ->assertCreated();
+    $this->withHeaders(['Authorization' => 'Bearer '.$token])
+        ->postJson($url, [...$payload, 'type' => 'file', 'mount_path' => '/etc/other.conf', 'content' => 'listen 8081;'])
+        ->assertCreated();
+    $this->withHeaders(['Authorization' => 'Bearer '.$token])
+        ->postJson($url, [...$payload, 'type' => 'file', 'is_host_file' => true, 'fs_path' => '/etc/hosts', 'mount_path' => '/etc/hosts'])
+        ->assertCreated();
+
+    expect($resource->fileStorages()->count())->toBe(4);
+})->with(['application', 'database', 'service']);
+
+test('changing the content of a mount outside the resource directory needs deploy', function (string $resourceType) {
+    [$url, , $resource] = storageHostPathApiTarget($resourceType);
+    $storage = LocalFileVolume::create([
+        'fs_path' => '/etc/zz/app.conf',
+        'mount_path' => '/etc/app.conf',
+        'content' => 'listen 8080;',
+        'resource_id' => $resource->id,
+        'resource_type' => $resource->getMorphClass(),
+    ]);
+
+    $this->withHeaders(['Authorization' => 'Bearer '.storageHostPathToken($this->admin, ['read', 'write'])])
+        ->patchJson($url, ['type' => 'file', 'uuid' => $storage->uuid, 'content' => 'ssh-ed25519 AAAA'])
+        ->assertForbidden()
+        ->assertJsonPath('message', 'Missing required permissions: deploy. A mount outside the resource directory needs a token with the deploy permission.');
+    expect($storage->fresh()->content)->toBe('listen 8080;');
+
+    $this->withHeaders(['Authorization' => 'Bearer '.storageHostPathToken($this->admin, ['read', 'write'])])
+        ->patchJson($url, ['type' => 'file', 'uuid' => $storage->uuid, 'mount_path' => '/etc/app2.conf', 'content' => 'listen 8080;'])
+        ->assertOk();
+    expect($storage->fresh()->mount_path)->toBe('/etc/app2.conf');
+
+    $this->withHeaders(['Authorization' => 'Bearer '.storageHostPathToken($this->admin, ['read', 'write', 'deploy'])])
+        ->patchJson($url, ['type' => 'file', 'uuid' => $storage->uuid, 'content' => 'listen 9090;'])
+        ->assertOk();
+    expect($storage->fresh()->content)->toBe('listen 9090;');
+})->with(['application', 'database', 'service']);

tests/Feature/VolumeBackupRecoveryBoundsTest.php

@@ -17,12 +17,15 @@ use App\Models\Server;
 use App\Models\StandaloneDocker;
 use App\Models\Team;
 use App\Models\User;
+use App\Notifications\Channels\DiscordChannel;
+use App\Notifications\VolumeBackup\RecoveryFailed;
 use Aws\Command;
 use Aws\S3\Exception\S3Exception;
 use Illuminate\Foundation\Testing\RefreshDatabase;
 use Illuminate\Support\Carbon;
 use Illuminate\Support\Facades\Cache;
 use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\Notification;
 use Illuminate\Support\Facades\Process;
 use Illuminate\Support\Facades\Queue;
 use Illuminate\Support\Facades\Storage;
@@ -33,6 +36,7 @@ uses(RefreshDatabase::class);
 
 beforeEach(function () {
     Server::flushIdentityMap();
+    Notification::fake();
     config(['broadcasting.default' => 'null']);
     InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));
 });
@@ -130,9 +134,22 @@ function makeRecoveryBoundsServerFunctional(Server $server): void
     ]);
 }
 
-it('doubles the recovery backoff and caps it at six hours', function () {
-    expect(collect(range(1, 9))->map(fn (int $attempts) => VolumeBackupRecoveryJob::backoffMinutes($attempts))->all())
-        ->toBe([5, 10, 20, 40, 80, 160, 320, 360, 360]);
+it('notifies the team once when recovery needs attention', function () {
+    $team = Team::factory()->create();
+    $team->discordNotificationSettings()->update(['discord_enabled' => true, 'backup_failure_discord_notifications' => true]);
+    $team->webhookNotificationSettings()->update(['webhook_enabled' => true, 'backup_failure_webhook_notifications' => false]);
+    [, , , $backup] = createRecoveryBoundsBackup($team);
+    $execution = createRecoveryBoundsExecution($backup);
+    failRecoveryBoundsS3Delete('AccessDenied');
+
+    VolumeBackupRecoveryJob::recoverWithBounds($execution);
+    VolumeBackupRecoveryJob::recoverWithBounds($execution);
+
+    Notification::assertSentToTimes($team, RecoveryFailed::class, 1);
+    Notification::assertSentTo($team, RecoveryFailed::class, fn (RecoveryFailed $notification, array $channels) => $notification->execution->is($execution)
+        && $channels === [DiscordChannel::class]
+        && str_contains($notification->toDiscord()->description, 'S3 credentials were rejected')
+        && str_contains((string) $notification->toMail()->render(), 'S3 credentials were rejected'));
 });
 
 it('records a categorized S3 cleanup failure without throwing or leaking the S3 message', function () {
@@ -148,11 +165,9 @@ it('records a categorized S3 cleanup failure without throwing or leaking the S3
     $execution->refresh();
     expect($execution->s3_cleanup_pending)->toBeTrue()
         ->and($execution->s3_storage_deleted)->toBeFalse()
-        ->and($execution->recovery_attempts)->toBe(1)
         ->and($execution->recovery_error)->toBe('s3_auth')
         ->and($execution->recovery_last_attempt_at->equalTo(now()))->toBeTrue()
-        ->and($execution->recovery_next_retry_at->equalTo(now()->addMinutes(5)))->toBeTrue()
-        ->and($execution->recovery_needs_attention)->toBeFalse()
+        ->and($execution->recovery_needs_attention)->toBeTrue()
         ->and($execution->message)->toBe('Worker timed out');
     $logger->shouldHaveReceived('warning')->once()->with(
         'Volume backup recovery failed',
@@ -162,38 +177,37 @@ it('records a categorized S3 cleanup failure without throwing or leaking the S3
     );
 });
 
-it('categorizes a missing S3 bucket and grows the backoff on each failure', function () {
+it('categorizes a missing S3 bucket and does not try again after the first failure', function () {
     Carbon::setTestNow('2026-10-03 12:00:00');
     [, , , $backup] = createRecoveryBoundsBackup(Team::factory()->create());
     $execution = createRecoveryBoundsExecution($backup);
-    failRecoveryBoundsS3Delete('NoSuchBucket');
+    $disk = Mockery::mock();
+    $disk->shouldReceive('delete')->once()->andThrow(UnableToDeleteFile::atLocation('partial.tar.gz', '', new S3Exception(
+        'The specified bucket does not exist',
+        new Command('DeleteObject'),
+        ['code' => 'NoSuchBucket'],
+    )));
+    Storage::shouldReceive('build')->once()->andReturn($disk);
 
     VolumeBackupRecoveryJob::recoverWithBounds($execution);
     VolumeBackupRecoveryJob::recoverWithBounds($execution);
 
     $execution->refresh();
     expect($execution->recovery_error)->toBe('s3_bucket')
-        ->and($execution->recovery_attempts)->toBe(2)
-        ->and($execution->recovery_next_retry_at->equalTo(now()->addMinutes(10)))->toBeTrue();
+        ->and($execution->recovery_needs_attention)->toBeTrue();
 });
 
-it('needs attention after the maximum attempts and stops automatic recovery', function () {
+it('stops automatic recovery after the first failure that is not an unreachable server', function () {
     Carbon::setTestNow('2026-10-03 12:00:00');
     config(['constants.coolify.self_hosted' => true]);
     Queue::fake();
     [, , , $backup] = createRecoveryBoundsBackup(Team::factory()->create());
-    $execution = createRecoveryBoundsExecution($backup, [
-        'recovery_attempts' => VolumeBackupRecoveryJob::MAX_ATTEMPTS - 1,
-    ]);
+    $execution = createRecoveryBoundsExecution($backup);
     failRecoveryBoundsS3Delete('AccessDenied');
 
-    VolumeBackupRecoveryJob::recoverWithBounds($execution);
     VolumeBackupRecoveryJob::recoverWithBounds($execution);
 
-    $execution->refresh();
-    expect($execution->recovery_needs_attention)->toBeTrue()
-        ->and($execution->recovery_attempts)->toBe(VolumeBackupRecoveryJob::MAX_ATTEMPTS)
-        ->and($execution->recovery_next_retry_at)->toBeNull();
+    expect($execution->fresh()->recovery_needs_attention)->toBeTrue();
 
     Carbon::setTestNow('2026-10-04 12:00:00');
     (new ScheduledJobManager)->handle();
@@ -201,20 +215,27 @@ it('needs attention after the maximum attempts and stops automatic recovery', fu
     Queue::assertNotPushed(VolumeBackupRecoveryJob::class);
 });
 
-it('waits for the backoff before dispatching recovery again', function () {
+it('dispatches container recovery again only when the server is functional, once per execution', function () {
     Carbon::setTestNow('2026-10-03 12:00:00');
     config(['constants.coolify.self_hosted' => true]);
+    Storage::fake('ssh-mux');
     Queue::fake();
-    [, , , $backup] = createRecoveryBoundsBackup(Team::factory()->create());
+    [, , $server, $backup] = createRecoveryBoundsBackup(Team::factory()->create());
+    $server->settings()->update(['is_reachable' => false]);
     createRecoveryBoundsExecution($backup, [
-        'recovery_attempts' => 3,
-        'recovery_next_retry_at' => now()->addMinutes(20),
+        'stop_recovery_pending' => true,
+        'recovery_error' => 'server_unreachable',
+        'recovery_last_attempt_at' => now(),
     ]);
 
+    (new ScheduledJobManager)->handle();
+    Carbon::setTestNow('2026-10-04 12:00:00');
     (new ScheduledJobManager)->handle();
     Queue::assertNotPushed(VolumeBackupRecoveryJob::class);
 
-    Carbon::setTestNow('2026-10-03 12:20:00');
+    makeRecoveryBoundsServerFunctional($server);
+    Server::flushIdentityMap();
+    (new ScheduledJobManager)->handle();
     (new ScheduledJobManager)->handle();
     Queue::assertPushed(VolumeBackupRecoveryJob::class, 1);
 });
@@ -242,16 +263,15 @@ it('cleans the S3 upload even when container recovery fails', function () {
         ->and($execution->s3_cleanup_pending)->toBeFalse()
         ->and($execution->s3_storage_deleted)->toBeTrue()
         ->and($execution->recovery_error)->toBe('server_unreachable')
-        ->and($execution->recovery_attempts)->toBe(1);
+        ->and($execution->recovery_needs_attention)->toBeFalse();
+    Notification::assertNothingSent();
 });
 
 it('resets the recovery state after a successful attempt', function () {
     [, , , $backup] = createRecoveryBoundsBackup(Team::factory()->create());
     $execution = createRecoveryBoundsExecution($backup, [
-        'recovery_attempts' => 4,
-        'recovery_error' => 's3_auth',
+        'recovery_error' => 'server_unreachable',
         'recovery_last_attempt_at' => now()->subHour(),
-        'recovery_next_retry_at' => now()->subMinute(),
     ]);
     $disk = Mockery::mock();
     $disk->shouldReceive('delete')->once()->andReturnTrue();
@@ -262,9 +282,9 @@ it('resets the recovery state after a successful attempt', function () {
     $execution->refresh();
     expect($execution->s3_cleanup_pending)->toBeFalse()
         ->and($execution->s3_storage_deleted)->toBeTrue()
-        ->and($execution->recovery_attempts)->toBe(0)
         ->and($execution->recovery_error)->toBeNull()
-        ->and($execution->recovery_next_retry_at)->toBeNull();
+        ->and($execution->recovery_last_attempt_at)->toBeNull()
+        ->and($execution->recovery_needs_attention)->toBeFalse();
 });
 
 it('only blocks new scheduled backups while stopped containers await recovery', function (array $pending, bool $dispatched) {
@@ -340,7 +360,6 @@ it('shows the recovery state and retries recovery manually', function () {
     signInForRecoveryBounds($this, $team);
     [$application, $volume, , $backup] = createRecoveryBoundsBackup($team);
     $execution = createRecoveryBoundsExecution($backup, [
-        'recovery_attempts' => VolumeBackupRecoveryJob::MAX_ATTEMPTS,
         'recovery_error' => 's3_auth',
         'recovery_needs_attention' => true,
     ]);
@@ -354,10 +373,9 @@ it('shows the recovery state and retries recovery manually', function () {
         ->assertSee('Recovery pending');
 
     $execution->refresh();
-    expect($execution->recovery_attempts)->toBe(0)
-        ->and($execution->recovery_needs_attention)->toBeFalse()
-        ->and($execution->recovery_next_retry_at)->toBeNull()
+    expect($execution->recovery_needs_attention)->toBeFalse()
         ->and(Cache::has(VolumeBackupRecoveryJob::dispatchCacheKey($execution->id)))->toBeFalse();
+    Queue::assertPushed(VolumeBackupRecoveryJob::class, 1);
     Queue::assertPushed(VolumeBackupRecoveryJob::class, fn (VolumeBackupRecoveryJob $job) => $job->execution->is($execution));
 });
 
@@ -365,7 +383,6 @@ it('prevents another team from retrying recovery', function () {
     Queue::fake();
     [$application, $volume, , $backup] = createRecoveryBoundsBackup(Team::factory()->create());
     $execution = createRecoveryBoundsExecution($backup, [
-        'recovery_attempts' => VolumeBackupRecoveryJob::MAX_ATTEMPTS,
         'recovery_needs_attention' => true,
     ]);
     $otherTeam = Team::factory()->create();
@@ -378,7 +395,6 @@ it('prevents another team from retrying recovery', function () {
         ->call('retryRecovery', $execution->id)
         ->assertDispatched('error');
 
-    expect($execution->fresh()->recovery_needs_attention)->toBeTrue()
-        ->and($execution->fresh()->recovery_attempts)->toBe(VolumeBackupRecoveryJob::MAX_ATTEMPTS);
+    expect($execution->fresh()->recovery_needs_attention)->toBeTrue();
     Queue::assertNotPushed(VolumeBackupRecoveryJob::class);
 });

tests/Feature/VolumeBackupTest.php

@@ -2676,6 +2676,7 @@ it('dispatches pending recovery for an execution at most once every five minutes
     Queue::fake();
     $team = Team::factory()->create();
     [$application, $volume] = createVolumeBackupApplication($team);
+    $application->destination->server->settings()->update(['is_reachable' => true, 'is_usable' => true, 'force_disabled' => false]);
     $backup = $volume->scheduledBackups()->create([
         'team_id' => $team->id,
         'frequency' => 'daily',

tests/Unit/DockerComposeInjectionCompatibilityTest.php

@@ -229,3 +229,44 @@ test('Compose injection payloads are rejected', function (string $compose, strin
     'mixed network name with an unsafe default' => ["services:\n  web:\n    image: nginx\nnetworks:\n  edge:\n    name: 'app-\${ENV:-x y}'\n", 'Invalid Docker Compose network name field'],
     'network name variable with an unsafe default' => ["services:\n  web:\n    image: nginx\nnetworks:\n  edge:\n    name: '\${NET:-x\$(touch /tmp/pwned)}'\n", 'Invalid Docker Compose network name field'],
 ]);
+
+test('service network list entries accept the same variables as network names', function (string $network) {
+    $compose = "services:\n  web:\n    image: nginx\n    networks:\n      - '{$network}'\nnetworks:\n  proxy:\n    external: true\n";
+
+    expect(fn () => validateDockerComposeForInjection($compose))->not->toThrow(Exception::class);
+})->with([
+    'variable with a default' => ['${NET:-proxy}'],
+    'variable with an unset-only default' => ['${NET-proxy}'],
+    'braced variable' => ['${NET}'],
+    'plain variable' => ['$NET'],
+    'variable mixed with text' => ['${COMPOSE_PROJECT_NAME}_default'],
+    'text with a variable default' => ['app-${ENV:-prod}'],
+]);
+
+test('unsafe service network list entries with variables are rejected', function (string $network) {
+    $compose = "services:\n  web:\n    image: nginx\n    networks:\n      - '{$network}'\n";
+
+    expect(fn () => validateDockerComposeForInjection($compose))->toThrow(Exception::class, 'Invalid Docker Compose service network');
+})->with([
+    'command substitution' => ['$(id)'],
+    'backticks' => ['`id`'],
+    'variable with command substitution' => ['${NET}$(id)'],
+    'default with command substitution' => ['${NET:-$(id)}'],
+    'default with a space' => ['${NET:-x y}'],
+    'default with backticks' => ['${NET:-`id`}'],
+]);
+
+test('a service network key with a variable is rejected because Compose does not interpolate keys', function () {
+    $compose = "services:\n  web:\n    image: nginx\n    networks:\n      '\${NET:-proxy}': {}\n";
+
+    expect(fn () => validateDockerComposeForInjection($compose))->toThrow(Exception::class, 'Invalid Docker Compose service network');
+});
+
+test('network errors name the service and the invalid value', function () {
+    $compose = "services:\n  web:\n    image: nginx\n    networks:\n      - 'net;id'\n";
+
+    expect(fn () => validateDockerComposeForInjection($compose))
+        ->toThrow(Exception::class, 'Invalid Docker Compose service network "net;id" in service web. Network names must start with an alphanumeric character and contain only alphanumeric characters, dots, hyphens, and underscores, and can use variables such as ${NETWORK:-default}.');
+    expect(fn () => validateDockerComposeForInjection("services:\n  web:\n    image: nginx\nnetworks:\n  'net;id': {}\n"))
+        ->toThrow(Exception::class, 'Invalid Docker Compose network name "net;id".');
+});