diff --git a/app/Actions/CoolifyTask/RunRemoteProcess.php b/app/Actions/CoolifyTask/RunRemoteProcess.php index 5b936686ce..90affa8db2 100644 --- a/app/Actions/CoolifyTask/RunRemoteProcess.php +++ b/app/Actions/CoolifyTask/RunRemoteProcess.php @@ -9,6 +9,7 @@ use App\Jobs\ApplicationDeploymentJob; use App\Models\Server; use App\Support\DatabaseImport\DatabaseImportCleanup; use App\Support\RemoteProcessCommand; +use App\Support\ResourceStartActivity; use App\Traits\BroadcastsToTeam; use Illuminate\Contracts\Process\ProcessResult; use Illuminate\Support\Facades\DB; @@ -92,6 +93,28 @@ class RunRemoteProcess $this->activity->save(); $processResult = $process->wait(); + + // A database import whose SSH command ended without the restore result keeps blocking the + // database until its restore is stopped. The stop event also cleans up, so the normal + // finish event must not run first. + if (array_key_exists($processResult->exitCode(), DatabaseImportCleanup::TRANSPORT_FAILURE_REASONS) + && $this->activity->properties->get('operation') === ResourceStartActivity::DATABASE_IMPORT_OPERATION + && blank(Activity::query()->whereKey($this->activity->getKey())->first()?->properties?->get(DatabaseImportCleanup::STOP_REQUESTED_PROPERTY)) + && DatabaseImportCleanup::stopAfterTransportFailure($this->activity, $processResult->exitCode())) { + $this->activity->properties = $this->activity->properties->merge([ + 'exitCode' => $processResult->exitCode(), + 'stdout' => $processResult->output(), + 'stderr' => $processResult->errorOutput(), + ]); + $this->activity->save(); + + if (! $this->ignore_errors) { + throw new \RuntimeException($processResult->errorOutput(), $processResult->exitCode()); + } + + return $processResult; + } + if ($this->activity->properties->get('status') === ProcessStatus::ERROR->value) { $status = ProcessStatus::ERROR; } else { diff --git a/app/Actions/Database/StartDatabaseImport.php b/app/Actions/Database/StartDatabaseImport.php index b91d63cc92..5e113d163c 100644 --- a/app/Actions/Database/StartDatabaseImport.php +++ b/app/Actions/Database/StartDatabaseImport.php @@ -31,8 +31,27 @@ class StartDatabaseImport public const LOCK_SECONDS = 1800; + /** + * Added to the SSH command timeout, so the remote restore ends (or times out) before the job. + * The job stays this far below the Horizon worker timeout too. + */ + public const JOB_TIMEOUT_MARGIN_SECONDS = 300; + public function __construct(private readonly DatabaseImportCommandBuilder $commands) {} + /** + * The CoolifyTask timeout of an import. A restore can run until the SSH command timeout, so the + * job must not time out before it. It stays below the Horizon worker timeout, which is below the + * queue retry_after, so the worker does not kill or run the job again while it runs. + */ + public static function jobTimeoutSeconds(): int + { + $timeout = (int) config('constants.ssh.command_timeout') + self::JOB_TIMEOUT_MARGIN_SECONDS; + $workerTimeout = (int) config('horizon.defaults.s6.timeout'); + + return $workerTimeout > 0 ? min($timeout, $workerTimeout - self::JOB_TIMEOUT_MARGIN_SECONDS) : $timeout; + } + public static function lockKey(string $resourceUuid): string { return "database-import:{$resourceUuid}"; @@ -157,13 +176,15 @@ class StartDatabaseImport $commandList[] = 'chmod +x '.escapeshellarg($scriptPath); $commandList[] = 'docker cp '.escapeshellarg($scriptPath).' '.escapeshellarg("{$container}:{$scriptPath}"); $commandList[] = 'rm -f '.escapeshellarg($scriptPath); - $commandList[] = 'docker exec '.escapeshellarg($container).' sh -c '.escapeshellarg($scriptPath); + // The restore reports its own 124 and 255 as 1: Coolify reads those exit codes of the SSH + // command as a local timeout or a lost connection and then stops the restore. + $commandList[] = 'docker exec '.escapeshellarg($container).' sh -c '.escapeshellarg('"$0"; status=$?; [ "$status" -ne 124 ] && [ "$status" -ne 255 ] || status=1; exit "$status"').' '.escapeshellarg($scriptPath); // The operation properties are set when the activity is created: the CoolifyTask job can // load and save the activity before a later update, which would drop them again. // The cleanup data (names and paths only, no credentials) lets Coolify stop and clean up // an import that it fails after a restart or because it is stale. - return remote_process($commandList, $server, type_uuid: $resource->uuid, model: $resource, callEventOnFinish: 'DatabaseImportFinished', callEventData: $cleanup, properties: [ + return remote_process($commandList, $server, type_uuid: $resource->uuid, model: $resource, callEventOnFinish: 'DatabaseImportFinished', callEventData: $cleanup, timeout: self::jobTimeoutSeconds(), properties: [ 'operation' => ResourceStartActivity::DATABASE_IMPORT_OPERATION, 'resource_kind' => $resource instanceof ServiceDatabase ? 'service_database' : 'standalone_database', 'operation_uuid' => $operation, diff --git a/app/Jobs/CoolifyTask.php b/app/Jobs/CoolifyTask.php index c19b61e19f..44739cd3f1 100755 --- a/app/Jobs/CoolifyTask.php +++ b/app/Jobs/CoolifyTask.php @@ -36,13 +36,19 @@ class CoolifyTask implements ShouldBeEncrypted, ShouldQueue /** * Create a new job instance. + * + * @param int|null $timeout Job timeout in seconds; null keeps the default above. */ public function __construct( public Activity $activity, public bool $ignore_errors, public $call_event_on_finish, public $call_event_data, + ?int $timeout = null, ) { + if ($timeout !== null) { + $this->timeout = $timeout; + } $this->onQueue('high'); } @@ -96,6 +102,14 @@ class CoolifyTask implements ShouldBeEncrypted, ShouldQueue 'trace' => $exception?->getTraceAsString(), ]); + // A database import whose restore can still run in the database container stays in progress + // until its stop has run, so it keeps blocking other operations on the database. + if (DatabaseImportCleanup::stopAfterTaskFailure($this->activity, $exception?->getMessage() ?: 'Job permanently failed')) { + RemoteProcessCommand::forget($this->activity); + + return; + } + // Update activity status to reflect permanent failure // A stopped database import already has the message that explains the stop; the process // error ("Terminated") would hide it. diff --git a/app/Listeners/CleanupDatabaseImport.php b/app/Listeners/CleanupDatabaseImport.php index a136aef55b..62f5c72404 100644 --- a/app/Listeners/CleanupDatabaseImport.php +++ b/app/Listeners/CleanupDatabaseImport.php @@ -111,6 +111,11 @@ SH; if ($server && $commands !== []) { instant_remote_process($commands, $server); } + + // A failed import task keeps blocking the database until its restore is stopped. + if ($claimed && ($event->data['stopRestore'] ?? false) === true) { + DatabaseImportCleanup::finishStop($operation); + } } catch (Throwable $exception) { if ($claimed) { DatabaseImportCleanup::release($operation); diff --git a/app/Support/DatabaseImport/DatabaseImportCleanup.php b/app/Support/DatabaseImport/DatabaseImportCleanup.php index 9cbd7900eb..9b7c7a72dc 100644 --- a/app/Support/DatabaseImport/DatabaseImportCleanup.php +++ b/app/Support/DatabaseImport/DatabaseImportCleanup.php @@ -2,7 +2,9 @@ namespace App\Support\DatabaseImport; +use App\Enums\ProcessStatus; use App\Events\DatabaseImportFinished; +use App\Support\ResourceStartActivity; use Illuminate\Support\Arr; use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Log; @@ -28,6 +30,23 @@ class DatabaseImportCleanup */ public const STOP_REQUESTED_PROPERTY = 'stop_requested_at'; + /** + * Why a failed import task was stopped. The activity gets this error when the stop has run. + */ + public const STOP_REASON_PROPERTY = 'stop_reason'; + + /** + * Exit codes of the local SSH command that do not come from the restore: ssh exits with 255 + * when the connection fails and the local `timeout` wrapper exits with 124. The restore + * command in the database container reports its own 124 and 255 as 1. + * + * @var array + */ + public const TRANSPORT_FAILURE_REASONS = [ + 124 => 'The import timed out.', + 255 => 'The connection to the server was lost.', + ]; + public const CLAIM_TTL_SECONDS = 7 * 24 * 3600; /** @@ -100,6 +119,81 @@ class DatabaseImportCleanup } } + /** + * Stop a database import whose CoolifyTask failed (for example it timed out) before the import + * reported its end, because the restore can still run in the database container. + * + * The activity stays in progress, so the import still blocks a start, restart, import or backup + * of the database. The cleanup listener marks it as failed after the stop has run. + * + * @return bool True when the import is stopped this way; the caller must not mark the activity as failed. + */ + public static function stopAfterTaskFailure(Activity $activity, string $reason): bool + { + if (data_get($activity, 'properties.operation') !== ResourceStartActivity::DATABASE_IMPORT_OPERATION) { + return false; + } + + $current = Activity::query()->find($activity->getKey()); + $status = data_get($current, 'properties.status'); + if (! $current || ! in_array($status, [ProcessStatus::QUEUED->value, ProcessStatus::IN_PROGRESS->value], true)) { + return false; + } + + if (self::stopRequested($current)) { + return true; + } + + $cleanup = self::stored($current); + if ($cleanup === null) { + return false; + } + + // Save the stop flag before the stop is queued, so the listener and a retried CoolifyTask see it. + $current->properties = $current->properties->merge([ + self::STOP_REQUESTED_PROPERTY => now()->toIso8601String(), + self::STOP_REASON_PROPERTY => $reason, + ]); + $current->save(); + $activity->properties = $current->properties; + + self::queueStop($cleanup); + + return true; + } + + /** + * Stop a database import whose SSH command ended without the result of the restore (the + * connection was lost or the local timeout ended it). The restore runs with `docker exec` + * and does not stop with the SSH session, so it is handled like a failed task. + * + * @return bool True when the import is stopped this way; the caller must keep it in progress. + */ + public static function stopAfterTransportFailure(Activity $activity, ?int $exitCode): bool + { + $reason = self::TRANSPORT_FAILURE_REASONS[$exitCode] ?? null; + + return $reason !== null && self::stopAfterTaskFailure($activity, $reason); + } + + /** + * Mark the imports of an operation as failed that wait for the stop of their failed task. + * Called by the cleanup listener after the stop has run. + */ + public static function finishStop(string $operationUuid): void + { + Activity::query() + ->where('properties->operation_uuid', $operationUuid) + ->where('properties->operation', ResourceStartActivity::DATABASE_IMPORT_OPERATION) + ->whereIn('properties->status', [ProcessStatus::QUEUED->value, ProcessStatus::IN_PROGRESS->value]) + ->get() + ->filter(fn (Activity $activity): bool => filled(data_get($activity, 'properties.'.self::STOP_REASON_PROPERTY))) + ->each(fn (Activity $activity) => ResourceStartActivity::markFailed( + $activity, + data_get($activity, 'properties.'.self::STOP_REASON_PROPERTY).' '.ResourceStartActivity::IMPORT_STOPPED_MESSAGE, + )); + } + public static function claimKey(string $operationUuid): string { return "database-import-cleanup:{$operationUuid}"; diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.php index 62c9ae7d8d..b583521221 100644 --- a/bootstrap/helpers/remoteProcess.php +++ b/bootstrap/helpers/remoteProcess.php @@ -28,6 +28,7 @@ function remote_process( $callEventOnFinish = null, $callEventData = null, array $properties = [], + ?int $timeout = null, ): Activity { $type = $type ?? ActivityTypes::INLINE->value; $command = $command instanceof Collection ? $command->toArray() : $command; @@ -72,6 +73,7 @@ function remote_process( ignore_errors: $ignore_errors, call_event_on_finish: $callEventOnFinish, call_event_data: $callEventData, + timeout: $timeout, )); $activity->refresh(); diff --git a/tests/Feature/DatabaseImportInterruptedCleanupTest.php b/tests/Feature/DatabaseImportInterruptedCleanupTest.php index fa2e41cfd0..0ef66fed33 100644 --- a/tests/Feature/DatabaseImportInterruptedCleanupTest.php +++ b/tests/Feature/DatabaseImportInterruptedCleanupTest.php @@ -1,6 +1,7 @@ set('cache.default', 'array'); config()->set('constants.ssh.mux_enabled', false); InstanceSettings::forceCreate(['id' => 0]); @@ -63,6 +67,10 @@ beforeEach(function () { Queue::fake(); }); +afterEach(function () { + Server::flushIdentityMap(); +}); + /** * @return array */ @@ -265,3 +273,214 @@ it('keeps the stop message when the stopped import job fails', function () { ->and(data_get($activity, 'properties.error'))->toContain('The import was stopped') ->and(data_get($activity, 'properties.failed_at'))->not->toBeNull(); }); + +test('an import task may run longer than the SSH command timeout while other remote tasks keep the default timeout', function (int $commandTimeout) { + config()->set('constants.ssh.command_timeout', $commandTimeout); + Process::swap(new ProcessFactory); + Process::fake([ + '*stat -c %s*' => Process::result('2048'), + '*od -An -tx1*' => Process::result(' 2d 2d 20 50 6f 73'), + '*' => Process::result(''), + ]); + + app(StartDatabaseImport::class)->handle($this->database, new DatabaseImportSource('server', path: '/srv/backups/app.sql'), $this->team->id); + remote_process(['echo ok'], $this->server); + + $timeouts = Queue::pushed(CoolifyTask::class)->mapWithKeys(fn (CoolifyTask $job) => [ + (data_get($job->activity, 'properties.operation') ?? 'other') => $job->timeout, + ]); + $workerTimeout = (int) config('horizon.defaults.s6.timeout'); + + expect($timeouts['other'])->toBe(600) + ->and($timeouts[ResourceStartActivity::DATABASE_IMPORT_OPERATION])->toBeGreaterThan(min($commandTimeout, $workerTimeout - 600)) + ->and($timeouts[ResourceStartActivity::DATABASE_IMPORT_OPERATION])->toBeLessThan($workerTimeout) + ->and($workerTimeout)->toBeLessThan((int) config('queue.connections.redis.retry_after')); +})->with([ + 'default SSH command timeout' => 3600, + 'SSH command timeout above the Horizon worker timeout' => 200000, +]); + +test('a failed import task stops the remote restore and blocks the database until the restore is stopped', function () { + $operation = (string) Str::uuid(); + $cleanup = storedImportCleanup($this, $operation); + $activity = runningImportActivity($this, ProcessStatus::IN_PROGRESS, $cleanup); + $job = new CoolifyTask($activity, ignore_errors: false, call_event_on_finish: 'DatabaseImportFinished', call_event_data: $cleanup); + + $job->failed(new RuntimeException('App\\Jobs\\CoolifyTask has timed out.')); + + assertStopQueuedOnce($cleanup); + expect(data_get($activity->refresh(), 'properties.status'))->toBe(ProcessStatus::IN_PROGRESS->value) + ->and(StartDatabase::operationInProgressError($this->database))->toBe(ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE); + + (new CleanupDatabaseImport)->handle(new DatabaseImportFinished([...$cleanup, 'stopRestore' => true])); + + Process::assertRan(fn ($process) => str_contains($process->command, "/tmp/restore_{$operation}") + && str_contains($process->command, 'Stopping the database import processes')); + $activity->refresh(); + expect(data_get($activity, 'properties.status'))->toBe(ProcessStatus::ERROR->value) + ->and(data_get($activity, 'properties.error'))->toBe('App\\Jobs\\CoolifyTask has timed out. '.ResourceStartActivity::IMPORT_STOPPED_MESSAGE) + ->and(StartDatabase::operationInProgressError($this->database))->toBeNull(); +}); + +test('a failed import task whose restore already ended is only cleaned up', function () { + $operation = (string) Str::uuid(); + $cleanup = storedImportCleanup($this, $operation); + $activity = runningImportActivity($this, ProcessStatus::ERROR, $cleanup); + $job = new CoolifyTask($activity, ignore_errors: false, call_event_on_finish: 'DatabaseImportFinished', call_event_data: $cleanup); + + $job->failed(new RuntimeException('psql: error')); + + Queue::assertPushed(CallQueuedListener::class, fn (CallQueuedListener $listener) => $listener->data[0]->data === $cleanup); + expect(data_get($activity->refresh(), 'properties.error'))->toBe('psql: error') + ->and(DatabaseImportCleanup::stopRequested($activity))->toBeFalse(); +}); + +/** + * A CoolifyTask of a running import whose SSH command ends with the given result. + * + * @return array{0: Activity, 1: CoolifyTask, 2: array} + */ +function importTaskEndingWith(object $test, int $exitCode, string $errorOutput): array +{ + $operation = (string) Str::uuid(); + $cleanup = storedImportCleanup($test, $operation); + $activity = runningImportActivity($test, ProcessStatus::IN_PROGRESS, $cleanup); + $activity->properties = $activity->properties->merge([ + 'server_uuid' => $test->server->uuid, + 'command' => "docker exec {$test->database->uuid} sh -c /tmp/restore_{$operation}.sh", + ]); + $activity->save(); + + Process::fake(fn () => Process::result(errorOutput: $errorOutput, exitCode: $exitCode)); + + return [$activity, new CoolifyTask($activity->fresh(), ignore_errors: false, call_event_on_finish: 'DatabaseImportFinished', call_event_data: $cleanup), $cleanup]; +} + +test('an import whose SSH command ends without the restore result stops the restore and blocks the database until it is stopped', function (int $exitCode, string $reason) { + [$activity, $job, $cleanup] = importTaskEndingWith($this, $exitCode, "Connection to 10.0.0.1 closed by remote host.\n"); + + // The queue calls failed() after the exception, because the task allows one exception. + try { + $job->handle(); + $this->fail('The task must fail.'); + } catch (RuntimeException $exception) { + $job->failed($exception); + } + + // Only the stop is queued; the normal cleanup without the stop must not run first. + assertStopQueuedOnce($cleanup); + $activity->refresh(); + expect(data_get($activity, 'properties.status'))->toBe(ProcessStatus::IN_PROGRESS->value) + ->and(data_get($activity, 'properties.exitCode'))->toBe($exitCode) + ->and(StartDatabase::operationInProgressError($this->database))->toBe(ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE); + + // The server is reachable again when the queued stop runs. + Process::swap(new ProcessFactory); + Process::fake(); + (new CleanupDatabaseImport)->handle(new DatabaseImportFinished([...$cleanup, 'stopRestore' => true])); + + Process::assertRan(fn ($process) => str_contains($process->command, 'Stopping the database import processes')); + $activity->refresh(); + expect(data_get($activity, 'properties.status'))->toBe(ProcessStatus::ERROR->value) + ->and(data_get($activity, 'properties.error'))->toBe("{$reason} ".ResourceStartActivity::IMPORT_STOPPED_MESSAGE) + ->and(RunRemoteProcess::decodeOutput($activity))->toContain($reason) + ->and(StartDatabase::operationInProgressError($this->database))->toBeNull(); +})->with([ + 'SSH connection lost (ssh exit 255)' => [255, 'The connection to the server was lost.'], + 'local SSH command timeout (timeout exit 124)' => [124, 'The import timed out.'], +]); + +test('an import whose restore fails by itself is failed and cleaned up without a stop', function () { + [$activity, $job, $cleanup] = importTaskEndingWith($this, 3, "psql: ERROR: syntax error\n"); + + expect(fn () => $job->handle())->toThrow(RuntimeException::class); + + Queue::assertPushed(CallQueuedListener::class, 1); + Queue::assertPushed(CallQueuedListener::class, fn (CallQueuedListener $listener) => $listener->data[0]->data === $cleanup); + $activity->refresh(); + expect(data_get($activity, 'properties.status'))->toBe(ProcessStatus::ERROR->value) + ->and(DatabaseImportCleanup::stopRequested($activity))->toBeFalse() + ->and(StartDatabase::operationInProgressError($this->database))->toBeNull(); +}); + +test('a lost import whose stop also fails stays blocking until the stale rule stops it again', function () { + config()->set('constants.ssh.max_retries', 1); + [$activity, $job, $cleanup] = importTaskEndingWith($this, 255, "Connection to 10.0.0.1 closed by remote host.\n"); + expect(fn () => $job->handle())->toThrow(RuntimeException::class); + + // The server is still unreachable, so the stop fails and the listener releases its claim for a retry. + expect(fn () => (new CleanupDatabaseImport)->handle(new DatabaseImportFinished([...$cleanup, 'stopRestore' => true]))) + ->toThrow(RuntimeException::class); + expect(data_get($activity->refresh(), 'properties.status'))->toBe(ProcessStatus::IN_PROGRESS->value) + ->and(StartDatabase::operationInProgressError($this->database))->toBe(ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE); + + $this->travel(ResourceStartActivity::importStaleAfterSeconds() + 60)->seconds(); + Queue::fake(); + + expect(StartDatabase::operationInProgressError($this->database))->toBeNull(); + assertStopQueuedOnce($cleanup); + expect(data_get($activity->refresh(), 'properties.error'))->toBe(ResourceStartActivity::STALE_MESSAGE.' '.ResourceStartActivity::IMPORT_STOPPED_MESSAGE); +}); + +test('a restore that exits like a lost SSH connection or timeout reports a normal failure, also for a non-root SSH user', function (string $user, int $restoreExitCode, int $expectedExitCode) { + $this->server->update(['user' => $user]); + Server::flushIdentityMap(); + Process::swap(new ProcessFactory); + Process::fake([ + '*stat -c %s*' => Process::result('2048'), + '*od -An -tx1*' => Process::result(' 2d 2d 20 50 6f 73'), + '*' => Process::result(''), + ]); + + app(StartDatabaseImport::class)->handle($this->database->fresh(), new DatabaseImportSource('server', path: '/srv/backups/app.sql'), $this->team->id); + + $task = Queue::pushed(CoolifyTask::class)->first(); + $restoreLine = collect(explode("\n", RemoteProcessCommand::read($task->activity)))->last(); + $scriptPath = data_get($task->activity, 'properties.'.DatabaseImportCleanup::PROPERTY.'.scriptPath'); + expect($restoreLine)->toContain($scriptPath); + + // Fake sudo and docker: `docker exec ...` runs the restore script on this machine. + $dir = sys_get_temp_dir().'/coolify-restore-exit-'.Str::random(8); + mkdir($dir); + file_put_contents("{$dir}/sudo", "#!/bin/sh\nexec \"\$@\"\n"); + file_put_contents("{$dir}/docker", "#!/bin/sh\n[ \"\$1\" = exec ] || exit 1\nshift 2\nexec \"\$@\"\n"); + file_put_contents($scriptPath, "echo restoring\nexit {$restoreExitCode}\n"); + chmod("{$dir}/sudo", 0755); + chmod("{$dir}/docker", 0755); + chmod($scriptPath, 0755); + + try { + $restore = new Symfony\Component\Process\Process(['bash', '-c', $restoreLine], env: ['PATH' => "{$dir}:".getenv('PATH')]); + $restore->run(); + + expect($restore->getExitCode())->toBe($expectedExitCode) + ->and($restore->getOutput())->toContain('restoring'); + } finally { + @unlink($scriptPath); + array_map('unlink', glob("{$dir}/*")); + rmdir($dir); + } +})->with([ + 'root, restore exits 255' => ['root', 255, 1], + 'root, restore exits 124' => ['root', 124, 1], + 'root, restore fails' => ['root', 3, 3], + 'root, restore succeeds' => ['root', 0, 0], + 'non-root, restore exits 255' => ['ubuntu', 255, 1], + 'non-root, restore fails' => ['ubuntu', 3, 3], +])->skip(fn () => PHP_OS_FAMILY !== 'Linux', 'Needs a POSIX shell.'); + +test('a normal remote task that loses its SSH connection fails as before without an import stop', function (int $exitCode) { + Queue::fake(); + Process::fake(fn () => Process::result(errorOutput: "Connection to 10.0.0.1 closed by remote host.\n", exitCode: $exitCode)); + $activity = remote_process(['docker ps'], $this->server, ignore_errors: true, callEventOnFinish: 'DatabaseImportFinished', callEventData: ['marker' => 'normal-task']); + + (new CoolifyTask($activity->fresh(), ignore_errors: true, call_event_on_finish: 'DatabaseImportFinished', call_event_data: ['marker' => 'normal-task']))->handle(); + + $activity->refresh(); + expect(data_get($activity, 'properties.status'))->toBe(ProcessStatus::ERROR->value) + ->and(data_get($activity, 'properties.exitCode'))->toBe($exitCode) + ->and(data_get($activity, 'properties.'.DatabaseImportCleanup::STOP_REQUESTED_PROPERTY))->toBeNull(); + // The normal finish event runs, without a stop. + Queue::assertPushed(CallQueuedListener::class, fn (CallQueuedListener $job) => ($job->data[0] ?? null) instanceof DatabaseImportFinished + && $job->data[0]->data === ['marker' => 'normal-task']); +})->with(['ssh exit 255' => 255, 'timeout exit 124' => 124]);