diff --git a/.changes/failed-image-scans-on-self-hosted.md b/.changes/failed-image-scans-on-self-hosted.md new file mode 100644 index 0000000000..dd59b43b4a --- /dev/null +++ b/.changes/failed-image-scans-on-self-hosted.md @@ -0,0 +1,4 @@ +--- +type: fixed +--- +Failed image scans on self-hosted instances, because the scanner templates were missing from the Coolify image diff --git a/.changes/the-reason-of-a-failed-image.md b/.changes/the-reason-of-a-failed-image.md new file mode 100644 index 0000000000..6be33a3520 --- /dev/null +++ b/.changes/the-reason-of-a-failed-image.md @@ -0,0 +1,6 @@ +--- +type: added +--- +The reason of a failed image scan in a `Show error log` section on the `Security` page and in the Laravel log + +- The error log shows the failed step, its exit code, and the end of the command output. Only users who can change the resource or server see it diff --git a/app/Actions/ImageScan/ScanHostPackages.php b/app/Actions/ImageScan/ScanHostPackages.php index 7c12035408..9251217163 100644 --- a/app/Actions/ImageScan/ScanHostPackages.php +++ b/app/Actions/ImageScan/ScanHostPackages.php @@ -55,14 +55,14 @@ class ScanHostPackages $packages = ScanImage::runStep($server, $commands->hostPackagesSize(), 'packages-size', self::TIMEOUT_DEFAULT); if ($packages['code'] !== 0 || $packages['size'] === null) { - throw new ScanFailed(ImageScanErrorCode::Unknown); + throw new ScanFailed(ImageScanErrorCode::Unknown, detail: "step packages-size exited with code {$packages['code']}"); } ScanImage::checkDiskSpace($commands, $packages['size'] * 1024); $sbom = ScanImage::runStep($server, $commands->sbomFromHost(), 'sbom', self::TIMEOUT_SBOM); $copy = $sbom['exit_codes']['copy'] ?? 2; if ($copy !== 0) { - throw new ScanFailed($copy === 1 ? ImageScanErrorCode::NoPackageDatabase : ImageScanErrorCode::Unknown); + throw new ScanFailed($copy === 1 ? ImageScanErrorCode::NoPackageDatabase : ImageScanErrorCode::Unknown, detail: "step copy exited with code {$copy}"); } ScanImage::assertSucceeded($sbom, 'sbom'); @@ -72,10 +72,11 @@ class ScanHostPackages $runningKernel = trim((string) instant_remote_process(['uname -r'], $server, throwError: false, timeout: self::TIMEOUT_DEFAULT)); $this->complete($server, $grype, $databaseBuiltAt, $runningKernel); } catch (ScanFailed $e) { - self::recordFailure($server, $e->errorCode, $e->context); + $e->log($server, 'the host packages'); + self::recordFailure($server, $e->errorCode, $e->errorContext()); } catch (Throwable $e) { report($e); - self::recordFailure($server, ImageScanErrorCode::Unknown); + self::recordFailure($server, ImageScanErrorCode::Unknown, (new ScanFailed(ImageScanErrorCode::Unknown, detail: $e::class.': '.$e->getMessage()))->errorContext()); } finally { instant_remote_process($commands->cleanup(), $server, throwError: false, timeout: self::TIMEOUT_DEFAULT); $lock->release(); diff --git a/app/Actions/ImageScan/ScanImage.php b/app/Actions/ImageScan/ScanImage.php index 17fa280a7d..08c6e602fd 100644 --- a/app/Actions/ImageScan/ScanImage.php +++ b/app/Actions/ImageScan/ScanImage.php @@ -122,8 +122,11 @@ class ScanImage try { ImageRef::parse($imageRef); - } catch (InvalidArgumentException) { - return $this->failBeforeRow($server, $resource, $imageRef, null, ImageScanErrorCode::Unknown, $context); + } catch (InvalidArgumentException $e) { + $failure = new ScanFailed(ImageScanErrorCode::Unknown, detail: "invalid image reference: {$e->getMessage()}"); + $failure->log($server, $imageRef); + + return $this->failBeforeRow($server, $resource, $imageRef, null, ImageScanErrorCode::Unknown, $failure->errorContext($context)); } $lock = self::serverLock($server); @@ -202,7 +205,10 @@ class ScanImage } if (preg_match('/\Asha256:[a-f0-9]{64}\z/', $imageId) !== 1) { - return array_map(fn (array $target) => $this->failBeforeRow($server, $target[0], $target[1], null, ImageScanErrorCode::Unknown, $context), $targets); + $failure = new ScanFailed(ImageScanErrorCode::Unknown, detail: 'invalid image ID'); + $failure->log($server, $imageId); + + return array_map(fn (array $target) => $this->failBeforeRow($server, $target[0], $target[1], null, ImageScanErrorCode::Unknown, $failure->errorContext($context)), $targets); } $lock = self::serverLock($server); @@ -326,13 +332,16 @@ class ScanImage return $this->complete($server, $resource, $imageRef, $deployment, $policy, $sbom, $this->report($grype), $image, $databaseBuiltAt, $isBaseline, $baseImageRef); } catch (ScanFailed $e) { - $errorContext = [...$context, ...$e->context]; + $e->log($server, $imageRef); + $errorContext = $e->errorContext($context); return $image === null ? $this->failBeforeRow($server, $resource, $imageRef, null, $e->errorCode, $errorContext) : $this->failAfterInspect($server, $resource, $imageRef, $deployment, $image, $e->errorCode, $errorContext); } catch (Throwable $e) { - $this->rememberError($resource, ImageScanErrorCode::Unknown, $context); + $failure = new ScanFailed(ImageScanErrorCode::Unknown, detail: $e::class.': '.$e->getMessage()); + $failure->log($server, $imageRef); + $this->rememberError($resource, ImageScanErrorCode::Unknown, $failure->errorContext($context)); throw $e; } finally { @@ -374,14 +383,17 @@ class ScanImage array_keys($targets), ); } catch (ScanFailed $e) { - $errorContext = [...$context, ...$e->context]; + $e->log($server, $imageId); + $errorContext = $e->errorContext($context); return array_map(fn (array $target) => $image === null ? $this->failBeforeRow($server, $target[0], $target[1], null, $e->errorCode, $errorContext) : $this->failAfterInspect($server, $target[0], $target[1], null, $image, $e->errorCode, $errorContext), $targets); } catch (Throwable $e) { + $failure = new ScanFailed(ImageScanErrorCode::Unknown, detail: $e::class.': '.$e->getMessage()); + $failure->log($server, $imageId); foreach ($resources as $resource) { - $this->rememberError($resource, ImageScanErrorCode::Unknown, $context); + $this->rememberError($resource, ImageScanErrorCode::Unknown, $failure->errorContext($context)); } throw $e; @@ -432,7 +444,7 @@ class ScanImage if ($builtAt === null) { self::updateDatabase($commands); - $builtAt = self::databaseStatus($commands) ?? throw new ScanFailed(ImageScanErrorCode::DbMissing); + $builtAt = self::databaseStatus($commands) ?? throw new ScanFailed(ImageScanErrorCode::DbMissing, detail: 'no valid vulnerability database after the update'); } if ($builtAt->lt(now()->subDays(self::DB_MAX_AGE_DAYS))) { @@ -457,7 +469,7 @@ class ScanImage { $result = self::runStep($commands->server, $commands->dbStatus(), 'db-status', self::TIMEOUT_DEFAULT); if (in_array($result['code'], self::EXIT_TIMEOUT, true)) { - throw new ScanFailed(ImageScanErrorCode::Timeout); + throw new ScanFailed(ImageScanErrorCode::Timeout, detail: "step db-status exited with code {$result['code']}"); } $status = $result['code'] === 0 ? json_decode($result['payload'], true) : null; @@ -489,7 +501,7 @@ class ScanImage $this->progress($resource, 'pull'); $result = self::runStep($commands->server, $commands->pull($imageRef), 'pull', self::TIMEOUT_PULL); if ($result['code'] !== 0) { - throw new ScanFailed(in_array($result['code'], self::EXIT_TIMEOUT, true) ? ImageScanErrorCode::Timeout : ImageScanErrorCode::PullFailed); + throw new ScanFailed(in_array($result['code'], self::EXIT_TIMEOUT, true) ? ImageScanErrorCode::Timeout : ImageScanErrorCode::PullFailed, detail: "step pull exited with code {$result['code']}"); } $this->progress($resource, 'inspect'); @@ -503,7 +515,7 @@ class ScanImage { $result = self::runStep($commands->server, $commands->inspect($imageRef), 'inspect', self::TIMEOUT_DEFAULT); if ($result['code'] !== 0) { - throw new ScanFailed(ImageScanErrorCode::ImageMissing); + throw new ScanFailed(ImageScanErrorCode::ImageMissing, detail: "step inspect exited with code {$result['code']}"); } $parts = explode('|', trim($result['payload'])); @@ -511,7 +523,7 @@ class ScanImage || preg_match('/\Asha256:([a-f0-9]{64})\z/', $parts[0], $id) !== 1 || preg_match('#\A[a-z0-9]+/[a-z0-9_]+\z#', $parts[2]) !== 1 || preg_match('/\A\d{1,18}\z/', $parts[3]) !== 1) { - throw new ScanFailed(ImageScanErrorCode::Unknown); + throw new ScanFailed(ImageScanErrorCode::Unknown, detail: 'unexpected docker image inspect output: '.ScanFailed::outputTail($result['payload'])); } $repoDigest = Str::afterLast($parts[1], '@'); @@ -536,12 +548,12 @@ class ScanImage { $output = instant_remote_process($commands->diskFree(), $commands->server, throwError: false, timeout: self::TIMEOUT_DEFAULT); if (preg_match('/(\d+) +(\d+)\s*\z/', (string) $output, $free) !== 1) { - throw new ScanFailed(ImageScanErrorCode::Unknown); + throw new ScanFailed(ImageScanErrorCode::Unknown, detail: 'could not read the free disk space: '.ScanFailed::outputTail($output)); } $needed = $imageSize * 2 + self::DISK_RESERVE_BYTES; if ((int) $free[1] < $needed || (int) $free[2] < $needed) { - throw new ScanFailed(ImageScanErrorCode::DiskSpace, ['n' => (int) ceil($needed / self::DISK_RESERVE_BYTES)]); + throw new ScanFailed(ImageScanErrorCode::DiskSpace, ['n' => (int) ceil($needed / self::DISK_RESERVE_BYTES)], "needed {$needed} bytes, free {$free[1]} and {$free[2]} bytes"); } } @@ -712,7 +724,7 @@ class ScanImage */ private function upsert(Model $resource, string $imageId, array $values): ImageScan { - $scan = $this->firstOrCreateScan($resource, $imageId, $values) ?? throw new ScanFailed(ImageScanErrorCode::Unknown); + $scan = $this->firstOrCreateScan($resource, $imageId, $values) ?? throw new ScanFailed(ImageScanErrorCode::Unknown, detail: "could not save the scan row of image {$imageId}"); if (! $scan->wasRecentlyCreated) { $scan->forceFill($values)->save(); @@ -920,7 +932,7 @@ class ScanImage $parsed = ScannerCommands::parseOutput($output); if ($output === null || trim($output) === '' || ! array_key_exists($step, $parsed['exit_codes'])) { - throw new ScanFailed(ImageScanErrorCode::Unknown); + throw new ScanFailed(ImageScanErrorCode::Unknown, detail: "step {$step} did not report an exit code (the command did not run or the SSH connection failed), output: ".ScanFailed::outputTail($output)); } return [ @@ -942,13 +954,15 @@ class ScanImage { $code = $result['exit_codes'][$step] ?? null; if ($code !== 0 && $step !== 'save' && in_array($step, $result['oom'], true)) { - throw new ScanFailed(ImageScanErrorCode::OutOfMemory); + throw new ScanFailed(ImageScanErrorCode::OutOfMemory, detail: "step {$step} was killed because it ran out of memory"); } if (in_array($code, self::EXIT_TIMEOUT, true)) { - throw new ScanFailed(ImageScanErrorCode::Timeout); + throw new ScanFailed(ImageScanErrorCode::Timeout, detail: "step {$step} exited with code {$code} (timeout)"); } if ($code !== 0) { - throw new ScanFailed(ImageScanErrorCode::Unknown); + $codes = json_encode($result['exit_codes']); + + throw new ScanFailed(ImageScanErrorCode::Unknown, detail: 'step '.$step.' exited with code '.($code ?? 'none').", all exit codes: {$codes}"); } } diff --git a/app/Livewire/ImageScan/Vulnerabilities.php b/app/Livewire/ImageScan/Vulnerabilities.php index 822ad99c42..13743c702d 100644 --- a/app/Livewire/ImageScan/Vulnerabilities.php +++ b/app/Livewire/ImageScan/Vulnerabilities.php @@ -722,23 +722,25 @@ class Vulnerabilities extends Component } /** - * The last scan errors of the owners, as their fixed messages. + * The last scan errors of the owners, as their fixed messages, with the error log for users who can update the resource. * * @param Collection $owners - * @return list + * @return list */ - private function errorMessages(Collection $owners): array + private function errorMessages(Collection $owners, bool $showDetail): array { $messages = []; foreach ($owners as $owner) { $error = Cache::get(ScanImage::lastErrorCacheKey($owner)); $code = is_array($error) ? ImageScanErrorCode::tryFrom((string) ($error['code'] ?? '')) : null; if ($code !== null) { - $messages[] = $code->message((array) ($error['context'] ?? [])); + $context = (array) ($error['context'] ?? []); + $detail = $showDetail && is_string($context['detail'] ?? null) ? $context['detail'] : null; + $messages[$code->message($context)."\n".$detail] = ['message' => $code->message($context), 'detail' => $detail]; } } - return array_values(array_unique($messages)); + return array_values($messages); } /** @@ -1003,7 +1005,7 @@ class Vulnerabilities extends Component 'scans' => $scans, 'progress' => $progress, 'checkpoints' => $this->checkpoints($progress), - 'errorMessages' => $this->errorMessages($owners), + 'errorMessages' => $this->errorMessages($owners, $canUpdate), 'card' => $security === null ? null : SecurityCard::for($resource, $security, $owners, $scans, $user, $imageStatuses, fn (Model $owner): RelevanceRules => $this->rulesFor($owner)), 'isApplication' => $resource instanceof Application, 'checkedAt' => $checkedAt === null ? null : Carbon::instance($checkedAt), diff --git a/app/Services/ImageScan/ScanFailed.php b/app/Services/ImageScan/ScanFailed.php index ed2e20b91a..82df1ad661 100644 --- a/app/Services/ImageScan/ScanFailed.php +++ b/app/Services/ImageScan/ScanFailed.php @@ -3,18 +3,58 @@ namespace App\Services\ImageScan; use App\Enums\ImageScanErrorCode; +use App\Models\Server; +use Illuminate\Support\Facades\Log; use RuntimeException; /** * Stops a scan with an error code. ScanImage catches it and stores the code on the scan row. + * The detail is for the log only: users see the fixed message of the error code. */ final class ScanFailed extends RuntimeException { + private const DETAIL_LIMIT = 1000; + /** * @param array $context */ - public function __construct(public readonly ImageScanErrorCode $errorCode, public readonly array $context = []) + public function __construct(public readonly ImageScanErrorCode $errorCode, public readonly array $context = [], public readonly ?string $detail = null) { - parent::__construct($errorCode->value); + parent::__construct($detail === null ? $errorCode->value : $errorCode->value.': '.mb_substr($detail, 0, self::DETAIL_LIMIT)); + } + + /** + * Writes the failure with its detail to the Laravel log, so an admin can find why a scan could not run. + */ + public function log(Server $server, string $target): void + { + Log::warning("Image scan of {$target} on server {$server->name} failed: {$this->getMessage()}", [ + 'server_uuid' => $server->uuid, + 'error_code' => $this->errorCode->value, + 'context' => $this->context, + ]); + } + + /** + * The values of the error message, plus the detail for the error log on the UI (`detail`; the message ignores it). + * + * @param array $context + * @return array + */ + public function errorContext(array $context = []): array + { + $context = [...$context, ...$this->context]; + + return $this->detail === null ? $context : [...$context, 'detail' => mb_substr($this->detail, 0, self::DETAIL_LIMIT)]; + } + + /** + * The last part of the raw command output, for the detail. + */ + public static function outputTail(?string $output): string + { + $output = trim((string) $output); + + return $output === '' ? '(no output)' : mb_substr($output, -self::DETAIL_LIMIT / 2); } } diff --git a/docker/production/Dockerfile b/docker/production/Dockerfile index 14ab1dd4f2..8a905d8c81 100644 --- a/docker/production/Dockerfile +++ b/docker/production/Dockerfile @@ -192,6 +192,7 @@ COPY --chown=www-data:www-data routes ./routes COPY --chown=www-data:www-data storage ./storage COPY --chown=www-data:www-data templates ./templates COPY --chown=www-data:www-data resources/views ./resources/views +COPY --chown=www-data:www-data resources/image-scan ./resources/image-scan COPY --chown=www-data:www-data artisan artisan COPY --chown=www-data:www-data openapi.yaml ./openapi.yaml COPY --chown=www-data:www-data changelogs/ ./changelogs/ diff --git a/resources/views/components/image-scan/error-log.blade.php b/resources/views/components/image-scan/error-log.blade.php new file mode 100644 index 0000000000..cd12ab8b5c --- /dev/null +++ b/resources/views/components/image-scan/error-log.blade.php @@ -0,0 +1,11 @@ +@props(['detail' => null]) + +@if (filled($detail)) +
+ + + Show error log + + 
{{ $detail }}
+
+@endif diff --git a/resources/views/livewire/image-scan/vulnerabilities.blade.php b/resources/views/livewire/image-scan/vulnerabilities.blade.php index dccad6f584..6f4f1e8c5c 100644 --- a/resources/views/livewire/image-scan/vulnerabilities.blade.php +++ b/resources/views/livewire/image-scan/vulnerabilities.blade.php @@ -51,8 +51,11 @@
- @foreach ($errorMessages as $message) - {{ $message }} + @foreach ($errorMessages as $error) + + {{ $error['message'] }} + + @endforeach @if ($isScanning) diff --git a/resources/views/livewire/server/security/host-vulnerabilities.blade.php b/resources/views/livewire/server/security/host-vulnerabilities.blade.php index 705caca3f7..eabaa1ea48 100644 --- a/resources/views/livewire/server/security/host-vulnerabilities.blade.php +++ b/resources/views/livewire/server/security/host-vulnerabilities.blade.php @@ -148,6 +148,9 @@ @if ($hostScanError) {{ $hostScanError }} + @can('update', $server) + + @endcan @endif @if ($hostScan?->eol)