From 1ee34a5ab2ec2cd4ca81ba0fa70252b22224afa1 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Fri, 2 Oct 2026 16:00:29 +0200 Subject: [PATCH] fix(database-import): stop the restore when the import task fails or loses SSH An import task can now run until the SSH command timeout (capped below the Horizon worker timeout). When the CoolifyTask fails, times out, or its SSH command ends with 124/255, Coolify stops the restore in the database container and keeps the import in progress until the stop has run, so the database stays blocked. The restore reports its own 124/255 as 1. Co-Authored-By: Claude Opus 5.5 --- app/Actions/CoolifyTask/RunRemoteProcess.php | 23 ++ app/Actions/Database/StartDatabaseImport.php | 25 +- app/Jobs/CoolifyTask.php | 14 ++ app/Listeners/CleanupDatabaseImport.php | 5 + .../DatabaseImport/DatabaseImportCleanup.php | 94 ++++++++ bootstrap/helpers/remoteProcess.php | 2 + .../DatabaseImportInterruptedCleanupTest.php | 219 ++++++++++++++++++ 7 files changed, 380 insertions(+), 2 deletions(-) diff --git a/app/Actions/CoolifyTask/RunRemoteProcess.php b/app/Actions/CoolifyTask/RunRemoteProcess.php index 5b936686ce..90affa8db2 100644 --- a/app/Actions/CoolifyTask/RunRemoteProcess.php +++ b/app/Actions/CoolifyTask/RunRemoteProcess.php @@ -9,6 +9,7 @@ use App\Jobs\ApplicationDeploymentJob; use App\Models\Server; use App\Support\DatabaseImport\DatabaseImportCleanup; use App\Support\RemoteProcessCommand; +use App\Support\ResourceStartActivity; use App\Traits\BroadcastsToTeam; use Illuminate\Contracts\Process\ProcessResult; use Illuminate\Support\Facades\DB; @@ -92,6 +93,28 @@ class RunRemoteProcess $this->activity->save(); $processResult = $process->wait(); + + // A database import whose SSH command ended without the restore result keeps blocking the + // database until its restore is stopped. The stop event also cleans up, so the normal + // finish event must not run first. + if (array_key_exists($processResult->exitCode(), DatabaseImportCleanup::TRANSPORT_FAILURE_REASONS) + && $this->activity->properties->get('operation') === ResourceStartActivity::DATABASE_IMPORT_OPERATION + && blank(Activity::query()->whereKey($this->activity->getKey())->first()?->properties?->get(DatabaseImportCleanup::STOP_REQUESTED_PROPERTY)) + && DatabaseImportCleanup::stopAfterTransportFailure($this->activity, $processResult->exitCode())) { + $this->activity->properties = $this->activity->properties->merge([ + 'exitCode' => $processResult->exitCode(), + 'stdout' => $processResult->output(), + 'stderr' => $processResult->errorOutput(), + ]); + $this->activity->save(); + + if (! $this->ignore_errors) { + throw new \RuntimeException($processResult->errorOutput(), $processResult->exitCode()); + } + + return $processResult; + } + if ($this->activity->properties->get('status') === ProcessStatus::ERROR->value) { $status = ProcessStatus::ERROR; } else { diff --git a/app/Actions/Database/StartDatabaseImport.php b/app/Actions/Database/StartDatabaseImport.php index b91d63cc92..5e113d163c 100644 --- a/app/Actions/Database/StartDatabaseImport.php +++ b/app/Actions/Database/StartDatabaseImport.php @@ -31,8 +31,27 @@ class StartDatabaseImport public const LOCK_SECONDS = 1800; + /** + * Added to the SSH command timeout, so the remote restore ends (or times out) before the job. + * The job stays this far below the Horizon worker timeout too. + */ + public const JOB_TIMEOUT_MARGIN_SECONDS = 300; + public function __construct(private readonly DatabaseImportCommandBuilder $commands) {} + /** + * The CoolifyTask timeout of an import. A restore can run until the SSH command timeout, so the + * job must not time out before it. It stays below the Horizon worker timeout, which is below the + * queue retry_after, so the worker does not kill or run the job again while it runs. + */ + public static function jobTimeoutSeconds(): int + { + $timeout = (int) config('constants.ssh.command_timeout') + self::JOB_TIMEOUT_MARGIN_SECONDS; + $workerTimeout = (int) config('horizon.defaults.s6.timeout'); + + return $workerTimeout > 0 ? min($timeout, $workerTimeout - self::JOB_TIMEOUT_MARGIN_SECONDS) : $timeout; + } + public static function lockKey(string $resourceUuid): string { return "database-import:{$resourceUuid}"; @@ -157,13 +176,15 @@ class StartDatabaseImport $commandList[] = 'chmod +x '.escapeshellarg($scriptPath); $commandList[] = 'docker cp '.escapeshellarg($scriptPath).' '.escapeshellarg("{$container}:{$scriptPath}"); $commandList[] = 'rm -f '.escapeshellarg($scriptPath); - $commandList[] = 'docker exec '.escapeshellarg($container).' sh -c '.escapeshellarg($scriptPath); + // The restore reports its own 124 and 255 as 1: Coolify reads those exit codes of the SSH + // command as a local timeout or a lost connection and then stops the restore. + $commandList[] = 'docker exec '.escapeshellarg($container).' sh -c '.escapeshellarg('"$0"; status=$?; [ "$status" -ne 124 ] && [ "$status" -ne 255 ] || status=1; exit "$status"').' '.escapeshellarg($scriptPath); // The operation properties are set when the activity is created: the CoolifyTask job can // load and save the activity before a later update, which would drop them again. // The cleanup data (names and paths only, no credentials) lets Coolify stop and clean up // an import that it fails after a restart or because it is stale. - return remote_process($commandList, $server, type_uuid: $resource->uuid, model: $resource, callEventOnFinish: 'DatabaseImportFinished', callEventData: $cleanup, properties: [ + return remote_process($commandList, $server, type_uuid: $resource->uuid, model: $resource, callEventOnFinish: 'DatabaseImportFinished', callEventData: $cleanup, timeout: self::jobTimeoutSeconds(), properties: [ 'operation' => ResourceStartActivity::DATABASE_IMPORT_OPERATION, 'resource_kind' => $resource instanceof ServiceDatabase ? 'service_database' : 'standalone_database', 'operation_uuid' => $operation, diff --git a/app/Jobs/CoolifyTask.php b/app/Jobs/CoolifyTask.php index c19b61e19f..44739cd3f1 100755 --- a/app/Jobs/CoolifyTask.php +++ b/app/Jobs/CoolifyTask.php @@ -36,13 +36,19 @@ class CoolifyTask implements ShouldBeEncrypted, ShouldQueue /** * Create a new job instance. + * + * @param int|null $timeout Job timeout in seconds; null keeps the default above. */ public function __construct( public Activity $activity, public bool $ignore_errors, public $call_event_on_finish, public $call_event_data, + ?int $timeout = null, ) { + if ($timeout !== null) { + $this->timeout = $timeout; + } $this->onQueue('high'); } @@ -96,6 +102,14 @@ class CoolifyTask implements ShouldBeEncrypted, ShouldQueue 'trace' => $exception?->getTraceAsString(), ]); + // A database import whose restore can still run in the database container stays in progress + // until its stop has run, so it keeps blocking other operations on the database. + if (DatabaseImportCleanup::stopAfterTaskFailure($this->activity, $exception?->getMessage() ?: 'Job permanently failed')) { + RemoteProcessCommand::forget($this->activity); + + return; + } + // Update activity status to reflect permanent failure // A stopped database import already has the message that explains the stop; the process // error ("Terminated") would hide it. diff --git a/app/Listeners/CleanupDatabaseImport.php b/app/Listeners/CleanupDatabaseImport.php index a136aef55b..62f5c72404 100644 --- a/app/Listeners/CleanupDatabaseImport.php +++ b/app/Listeners/CleanupDatabaseImport.php @@ -111,6 +111,11 @@ SH; if ($server && $commands !== []) { instant_remote_process($commands, $server); } + + // A failed import task keeps blocking the database until its restore is stopped. + if ($claimed && ($event->data['stopRestore'] ?? false) === true) { + DatabaseImportCleanup::finishStop($operation); + } } catch (Throwable $exception) { if ($claimed) { DatabaseImportCleanup::release($operation); diff --git a/app/Support/DatabaseImport/DatabaseImportCleanup.php b/app/Support/DatabaseImport/DatabaseImportCleanup.php index 9cbd7900eb..9b7c7a72dc 100644 --- a/app/Support/DatabaseImport/DatabaseImportCleanup.php +++ b/app/Support/DatabaseImport/DatabaseImportCleanup.php @@ -2,7 +2,9 @@ namespace App\Support\DatabaseImport; +use App\Enums\ProcessStatus; use App\Events\DatabaseImportFinished; +use App\Support\ResourceStartActivity; use Illuminate\Support\Arr; use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Log; @@ -28,6 +30,23 @@ class DatabaseImportCleanup */ public const STOP_REQUESTED_PROPERTY = 'stop_requested_at'; + /** + * Why a failed import task was stopped. The activity gets this error when the stop has run. + */ + public const STOP_REASON_PROPERTY = 'stop_reason'; + + /** + * Exit codes of the local SSH command that do not come from the restore: ssh exits with 255 + * when the connection fails and the local `timeout` wrapper exits with 124. The restore + * command in the database container reports its own 124 and 255 as 1. + * + * @var array + */ + public const TRANSPORT_FAILURE_REASONS = [ + 124 => 'The import timed out.', + 255 => 'The connection to the server was lost.', + ]; + public const CLAIM_TTL_SECONDS = 7 * 24 * 3600; /** @@ -100,6 +119,81 @@ class DatabaseImportCleanup } } + /** + * Stop a database import whose CoolifyTask failed (for example it timed out) before the import + * reported its end, because the restore can still run in the database container. + * + * The activity stays in progress, so the import still blocks a start, restart, import or backup + * of the database. The cleanup listener marks it as failed after the stop has run. + * + * @return bool True when the import is stopped this way; the caller must not mark the activity as failed. + */ + public static function stopAfterTaskFailure(Activity $activity, string $reason): bool + { + if (data_get($activity, 'properties.operation') !== ResourceStartActivity::DATABASE_IMPORT_OPERATION) { + return false; + } + + $current = Activity::query()->find($activity->getKey()); + $status = data_get($current, 'properties.status'); + if (! $current || ! in_array($status, [ProcessStatus::QUEUED->value, ProcessStatus::IN_PROGRESS->value], true)) { + return false; + } + + if (self::stopRequested($current)) { + return true; + } + + $cleanup = self::stored($current); + if ($cleanup === null) { + return false; + } + + // Save the stop flag before the stop is queued, so the listener and a retried CoolifyTask see it. + $current->properties = $current->properties->merge([ + self::STOP_REQUESTED_PROPERTY => now()->toIso8601String(), + self::STOP_REASON_PROPERTY => $reason, + ]); + $current->save(); + $activity->properties = $current->properties; + + self::queueStop($cleanup); + + return true; + } + + /** + * Stop a database import whose SSH command ended without the result of the restore (the + * connection was lost or the local timeout ended it). The restore runs with `docker exec` + * and does not stop with the SSH session, so it is handled like a failed task. + * + * @return bool True when the import is stopped this way; the caller must keep it in progress. + */ + public static function stopAfterTransportFailure(Activity $activity, ?int $exitCode): bool + { + $reason = self::TRANSPORT_FAILURE_REASONS[$exitCode] ?? null; + + return $reason !== null && self::stopAfterTaskFailure($activity, $reason); + } + + /** + * Mark the imports of an operation as failed that wait for the stop of their failed task. + * Called by the cleanup listener after the stop has run. + */ + public static function finishStop(string $operationUuid): void + { + Activity::query() + ->where('properties->operation_uuid', $operationUuid) + ->where('properties->operation', ResourceStartActivity::DATABASE_IMPORT_OPERATION) + ->whereIn('properties->status', [ProcessStatus::QUEUED->value, ProcessStatus::IN_PROGRESS->value]) + ->get() + ->filter(fn (Activity $activity): bool => filled(data_get($activity, 'properties.'.self::STOP_REASON_PROPERTY))) + ->each(fn (Activity $activity) => ResourceStartActivity::markFailed( + $activity, + data_get($activity, 'properties.'.self::STOP_REASON_PROPERTY).' '.ResourceStartActivity::IMPORT_STOPPED_MESSAGE, + )); + } + public static function claimKey(string $operationUuid): string { return "database-import-cleanup:{$operationUuid}"; diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.php index 62c9ae7d8d..b583521221 100644 --- a/bootstrap/helpers/remoteProcess.php +++ b/bootstrap/helpers/remoteProcess.php @@ -28,6 +28,7 @@ function remote_process( $callEventOnFinish = null, $callEventData = null, array $properties = [], + ?int $timeout = null, ): Activity { $type = $type ?? ActivityTypes::INLINE->value; $command = $command instanceof Collection ? $command->toArray() : $command; @@ -72,6 +73,7 @@ function remote_process( ignore_errors: $ignore_errors, call_event_on_finish: $callEventOnFinish, call_event_data: $callEventData, + timeout: $timeout, )); $activity->refresh(); diff --git a/tests/Feature/DatabaseImportInterruptedCleanupTest.php b/tests/Feature/DatabaseImportInterruptedCleanupTest.php index fa2e41cfd0..0ef66fed33 100644 --- a/tests/Feature/DatabaseImportInterruptedCleanupTest.php +++ b/tests/Feature/DatabaseImportInterruptedCleanupTest.php @@ -1,6 +1,7 @@ set('cache.default', 'array'); config()->set('constants.ssh.mux_enabled', false); InstanceSettings::forceCreate(['id' => 0]); @@ -63,6 +67,10 @@ beforeEach(function () { Queue::fake(); }); +afterEach(function () { + Server::flushIdentityMap(); +}); + /** * @return array */ @@ -265,3 +273,214 @@ it('keeps the stop message when the stopped import job fails', function () { ->and(data_get($activity, 'properties.error'))->toContain('The import was stopped') ->and(data_get($activity, 'properties.failed_at'))->not->toBeNull(); }); + +test('an import task may run longer than the SSH command timeout while other remote tasks keep the default timeout', function (int $commandTimeout) { + config()->set('constants.ssh.command_timeout', $commandTimeout); + Process::swap(new ProcessFactory); + Process::fake([ + '*stat -c %s*' => Process::result('2048'), + '*od -An -tx1*' => Process::result(' 2d 2d 20 50 6f 73'), + '*' => Process::result(''), + ]); + + app(StartDatabaseImport::class)->handle($this->database, new DatabaseImportSource('server', path: '/srv/backups/app.sql'), $this->team->id); + remote_process(['echo ok'], $this->server); + + $timeouts = Queue::pushed(CoolifyTask::class)->mapWithKeys(fn (CoolifyTask $job) => [ + (data_get($job->activity, 'properties.operation') ?? 'other') => $job->timeout, + ]); + $workerTimeout = (int) config('horizon.defaults.s6.timeout'); + + expect($timeouts['other'])->toBe(600) + ->and($timeouts[ResourceStartActivity::DATABASE_IMPORT_OPERATION])->toBeGreaterThan(min($commandTimeout, $workerTimeout - 600)) + ->and($timeouts[ResourceStartActivity::DATABASE_IMPORT_OPERATION])->toBeLessThan($workerTimeout) + ->and($workerTimeout)->toBeLessThan((int) config('queue.connections.redis.retry_after')); +})->with([ + 'default SSH command timeout' => 3600, + 'SSH command timeout above the Horizon worker timeout' => 200000, +]); + +test('a failed import task stops the remote restore and blocks the database until the restore is stopped', function () { + $operation = (string) Str::uuid(); + $cleanup = storedImportCleanup($this, $operation); + $activity = runningImportActivity($this, ProcessStatus::IN_PROGRESS, $cleanup); + $job = new CoolifyTask($activity, ignore_errors: false, call_event_on_finish: 'DatabaseImportFinished', call_event_data: $cleanup); + + $job->failed(new RuntimeException('App\\Jobs\\CoolifyTask has timed out.')); + + assertStopQueuedOnce($cleanup); + expect(data_get($activity->refresh(), 'properties.status'))->toBe(ProcessStatus::IN_PROGRESS->value) + ->and(StartDatabase::operationInProgressError($this->database))->toBe(ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE); + + (new CleanupDatabaseImport)->handle(new DatabaseImportFinished([...$cleanup, 'stopRestore' => true])); + + Process::assertRan(fn ($process) => str_contains($process->command, "/tmp/restore_{$operation}") + && str_contains($process->command, 'Stopping the database import processes')); + $activity->refresh(); + expect(data_get($activity, 'properties.status'))->toBe(ProcessStatus::ERROR->value) + ->and(data_get($activity, 'properties.error'))->toBe('App\\Jobs\\CoolifyTask has timed out. '.ResourceStartActivity::IMPORT_STOPPED_MESSAGE) + ->and(StartDatabase::operationInProgressError($this->database))->toBeNull(); +}); + +test('a failed import task whose restore already ended is only cleaned up', function () { + $operation = (string) Str::uuid(); + $cleanup = storedImportCleanup($this, $operation); + $activity = runningImportActivity($this, ProcessStatus::ERROR, $cleanup); + $job = new CoolifyTask($activity, ignore_errors: false, call_event_on_finish: 'DatabaseImportFinished', call_event_data: $cleanup); + + $job->failed(new RuntimeException('psql: error')); + + Queue::assertPushed(CallQueuedListener::class, fn (CallQueuedListener $listener) => $listener->data[0]->data === $cleanup); + expect(data_get($activity->refresh(), 'properties.error'))->toBe('psql: error') + ->and(DatabaseImportCleanup::stopRequested($activity))->toBeFalse(); +}); + +/** + * A CoolifyTask of a running import whose SSH command ends with the given result. + * + * @return array{0: Activity, 1: CoolifyTask, 2: array} + */ +function importTaskEndingWith(object $test, int $exitCode, string $errorOutput): array +{ + $operation = (string) Str::uuid(); + $cleanup = storedImportCleanup($test, $operation); + $activity = runningImportActivity($test, ProcessStatus::IN_PROGRESS, $cleanup); + $activity->properties = $activity->properties->merge([ + 'server_uuid' => $test->server->uuid, + 'command' => "docker exec {$test->database->uuid} sh -c /tmp/restore_{$operation}.sh", + ]); + $activity->save(); + + Process::fake(fn () => Process::result(errorOutput: $errorOutput, exitCode: $exitCode)); + + return [$activity, new CoolifyTask($activity->fresh(), ignore_errors: false, call_event_on_finish: 'DatabaseImportFinished', call_event_data: $cleanup), $cleanup]; +} + +test('an import whose SSH command ends without the restore result stops the restore and blocks the database until it is stopped', function (int $exitCode, string $reason) { + [$activity, $job, $cleanup] = importTaskEndingWith($this, $exitCode, "Connection to 10.0.0.1 closed by remote host.\n"); + + // The queue calls failed() after the exception, because the task allows one exception. + try { + $job->handle(); + $this->fail('The task must fail.'); + } catch (RuntimeException $exception) { + $job->failed($exception); + } + + // Only the stop is queued; the normal cleanup without the stop must not run first. + assertStopQueuedOnce($cleanup); + $activity->refresh(); + expect(data_get($activity, 'properties.status'))->toBe(ProcessStatus::IN_PROGRESS->value) + ->and(data_get($activity, 'properties.exitCode'))->toBe($exitCode) + ->and(StartDatabase::operationInProgressError($this->database))->toBe(ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE); + + // The server is reachable again when the queued stop runs. + Process::swap(new ProcessFactory); + Process::fake(); + (new CleanupDatabaseImport)->handle(new DatabaseImportFinished([...$cleanup, 'stopRestore' => true])); + + Process::assertRan(fn ($process) => str_contains($process->command, 'Stopping the database import processes')); + $activity->refresh(); + expect(data_get($activity, 'properties.status'))->toBe(ProcessStatus::ERROR->value) + ->and(data_get($activity, 'properties.error'))->toBe("{$reason} ".ResourceStartActivity::IMPORT_STOPPED_MESSAGE) + ->and(RunRemoteProcess::decodeOutput($activity))->toContain($reason) + ->and(StartDatabase::operationInProgressError($this->database))->toBeNull(); +})->with([ + 'SSH connection lost (ssh exit 255)' => [255, 'The connection to the server was lost.'], + 'local SSH command timeout (timeout exit 124)' => [124, 'The import timed out.'], +]); + +test('an import whose restore fails by itself is failed and cleaned up without a stop', function () { + [$activity, $job, $cleanup] = importTaskEndingWith($this, 3, "psql: ERROR: syntax error\n"); + + expect(fn () => $job->handle())->toThrow(RuntimeException::class); + + Queue::assertPushed(CallQueuedListener::class, 1); + Queue::assertPushed(CallQueuedListener::class, fn (CallQueuedListener $listener) => $listener->data[0]->data === $cleanup); + $activity->refresh(); + expect(data_get($activity, 'properties.status'))->toBe(ProcessStatus::ERROR->value) + ->and(DatabaseImportCleanup::stopRequested($activity))->toBeFalse() + ->and(StartDatabase::operationInProgressError($this->database))->toBeNull(); +}); + +test('a lost import whose stop also fails stays blocking until the stale rule stops it again', function () { + config()->set('constants.ssh.max_retries', 1); + [$activity, $job, $cleanup] = importTaskEndingWith($this, 255, "Connection to 10.0.0.1 closed by remote host.\n"); + expect(fn () => $job->handle())->toThrow(RuntimeException::class); + + // The server is still unreachable, so the stop fails and the listener releases its claim for a retry. + expect(fn () => (new CleanupDatabaseImport)->handle(new DatabaseImportFinished([...$cleanup, 'stopRestore' => true]))) + ->toThrow(RuntimeException::class); + expect(data_get($activity->refresh(), 'properties.status'))->toBe(ProcessStatus::IN_PROGRESS->value) + ->and(StartDatabase::operationInProgressError($this->database))->toBe(ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE); + + $this->travel(ResourceStartActivity::importStaleAfterSeconds() + 60)->seconds(); + Queue::fake(); + + expect(StartDatabase::operationInProgressError($this->database))->toBeNull(); + assertStopQueuedOnce($cleanup); + expect(data_get($activity->refresh(), 'properties.error'))->toBe(ResourceStartActivity::STALE_MESSAGE.' '.ResourceStartActivity::IMPORT_STOPPED_MESSAGE); +}); + +test('a restore that exits like a lost SSH connection or timeout reports a normal failure, also for a non-root SSH user', function (string $user, int $restoreExitCode, int $expectedExitCode) { + $this->server->update(['user' => $user]); + Server::flushIdentityMap(); + Process::swap(new ProcessFactory); + Process::fake([ + '*stat -c %s*' => Process::result('2048'), + '*od -An -tx1*' => Process::result(' 2d 2d 20 50 6f 73'), + '*' => Process::result(''), + ]); + + app(StartDatabaseImport::class)->handle($this->database->fresh(), new DatabaseImportSource('server', path: '/srv/backups/app.sql'), $this->team->id); + + $task = Queue::pushed(CoolifyTask::class)->first(); + $restoreLine = collect(explode("\n", RemoteProcessCommand::read($task->activity)))->last(); + $scriptPath = data_get($task->activity, 'properties.'.DatabaseImportCleanup::PROPERTY.'.scriptPath'); + expect($restoreLine)->toContain($scriptPath); + + // Fake sudo and docker: `docker exec ...` runs the restore script on this machine. + $dir = sys_get_temp_dir().'/coolify-restore-exit-'.Str::random(8); + mkdir($dir); + file_put_contents("{$dir}/sudo", "#!/bin/sh\nexec \"\$@\"\n"); + file_put_contents("{$dir}/docker", "#!/bin/sh\n[ \"\$1\" = exec ] || exit 1\nshift 2\nexec \"\$@\"\n"); + file_put_contents($scriptPath, "echo restoring\nexit {$restoreExitCode}\n"); + chmod("{$dir}/sudo", 0755); + chmod("{$dir}/docker", 0755); + chmod($scriptPath, 0755); + + try { + $restore = new Symfony\Component\Process\Process(['bash', '-c', $restoreLine], env: ['PATH' => "{$dir}:".getenv('PATH')]); + $restore->run(); + + expect($restore->getExitCode())->toBe($expectedExitCode) + ->and($restore->getOutput())->toContain('restoring'); + } finally { + @unlink($scriptPath); + array_map('unlink', glob("{$dir}/*")); + rmdir($dir); + } +})->with([ + 'root, restore exits 255' => ['root', 255, 1], + 'root, restore exits 124' => ['root', 124, 1], + 'root, restore fails' => ['root', 3, 3], + 'root, restore succeeds' => ['root', 0, 0], + 'non-root, restore exits 255' => ['ubuntu', 255, 1], + 'non-root, restore fails' => ['ubuntu', 3, 3], +])->skip(fn () => PHP_OS_FAMILY !== 'Linux', 'Needs a POSIX shell.'); + +test('a normal remote task that loses its SSH connection fails as before without an import stop', function (int $exitCode) { + Queue::fake(); + Process::fake(fn () => Process::result(errorOutput: "Connection to 10.0.0.1 closed by remote host.\n", exitCode: $exitCode)); + $activity = remote_process(['docker ps'], $this->server, ignore_errors: true, callEventOnFinish: 'DatabaseImportFinished', callEventData: ['marker' => 'normal-task']); + + (new CoolifyTask($activity->fresh(), ignore_errors: true, call_event_on_finish: 'DatabaseImportFinished', call_event_data: ['marker' => 'normal-task']))->handle(); + + $activity->refresh(); + expect(data_get($activity, 'properties.status'))->toBe(ProcessStatus::ERROR->value) + ->and(data_get($activity, 'properties.exitCode'))->toBe($exitCode) + ->and(data_get($activity, 'properties.'.DatabaseImportCleanup::STOP_REQUESTED_PROPERTY))->toBeNull(); + // The normal finish event runs, without a stop. + Queue::assertPushed(CallQueuedListener::class, fn (CallQueuedListener $job) => ($job->data[0] ?? null) instanceof DatabaseImportFinished + && $job->data[0]->data === ['marker' => 'normal-task']); +})->with(['ssh exit 255' => 255, 'timeout exit 124' => 124]);