diff --git a/.changes/image-scans-of-large-images-that.md b/.changes/image-scans-of-large-images-that.md new file mode 100644 index 0000000000..ba3071e95b --- /dev/null +++ b/.changes/image-scans-of-large-images-that.md @@ -0,0 +1,6 @@ +--- +type: fixed +--- +Image scans of large images that stopped because the scanner ran out of memory + +- The scanner frees memory sooner when it gets close to its limit. Set a higher limit with `Scanner memory limit` on the `Vulnerabilities` page of a server, or with `image_scan_memory_mb` in the server image scanning API. The default stays 1 GB diff --git a/app/Enums/ImageScanErrorCode.php b/app/Enums/ImageScanErrorCode.php index 4d137cdbd4..c34055f983 100644 --- a/app/Enums/ImageScanErrorCode.php +++ b/app/Enums/ImageScanErrorCode.php @@ -35,7 +35,7 @@ enum ImageScanErrorCode: string self::DbTooOld => "The vulnerability database is {$n} days old. {$server} cannot reach grype.anchore.io.", self::DbMissing => "The vulnerability database is not on {$server} yet. Wait for the first update to finish or check that {$server} can reach grype.anchore.io.", self::Timeout => 'The scan took too long and was stopped.', - self::OutOfMemory => 'The scanner ran out of memory and was stopped. It can use up to 1 GB, and the server must have that much free memory.', + self::OutOfMemory => "The scanner ran out of memory and was stopped. Raise the scanner memory limit on the Vulnerabilities page of {$server}, and make sure that {$server} has that much free memory.", self::LockBusy => "Another scan is still running on {$server}.", self::RegistryAuth => "Coolify cannot read this image from its registry. Log in to the registry on {$server}.", self::BuildServerOff => 'Image scanning is off on the build server.', diff --git a/app/Http/Controllers/Api/ServerImageScanningController.php b/app/Http/Controllers/Api/ServerImageScanningController.php index 080dbf2ec7..f2de7bee64 100644 --- a/app/Http/Controllers/Api/ServerImageScanningController.php +++ b/app/Http/Controllers/Api/ServerImageScanningController.php @@ -6,19 +6,21 @@ use App\Actions\ImageScan\SetServerImageScanning; use App\Http\Controllers\Controller; use App\Jobs\PrepareImageScannerJob; use App\Models\Server; +use App\Models\ServerSetting; use App\Services\ImageScan\ImageScanningRefused; use Carbon\CarbonImmutable; use Illuminate\Http\JsonResponse; use Illuminate\Http\Request; use Illuminate\Support\Facades\Cache; +use Illuminate\Validation\Rule; use OpenApi\Attributes as OA; class ServerImageScanningController extends Controller { - private const ALLOWED_FIELDS = ['is_image_scan_enabled']; + private const ALLOWED_FIELDS = ['is_image_scan_enabled', 'image_scan_memory_mb']; /** - * @return array{is_image_scan_enabled: bool, scanner_state: ?string, scanner_failed_step: ?string, scanner_built_at: ?string, scanner_disk_used_bytes: ?int} + * @return array{is_image_scan_enabled: bool, image_scan_memory_mb: int, scanner_state: ?string, scanner_failed_step: ?string, scanner_built_at: ?string, scanner_disk_used_bytes: ?int} */ private function transform(Server $server): array { @@ -29,6 +31,7 @@ class ServerImageScanningController extends Controller return [ 'is_image_scan_enabled' => (bool) $server->settings->is_image_scan_enabled, + 'image_scan_memory_mb' => (int) $server->settings->image_scan_memory_mb, 'scanner_state' => is_string($state) ? $state : null, 'scanner_failed_step' => is_string($failedStep) ? $failedStep : null, 'scanner_built_at' => is_string($builtAt) ? CarbonImmutable::parse($builtAt)->toIso8601String() : null, @@ -60,6 +63,7 @@ class ServerImageScanningController extends Controller content: new OA\JsonContent( properties: [ new OA\Property(property: 'is_image_scan_enabled', type: 'boolean', description: 'Whether the server scans the images of its resources.'), + new OA\Property(property: 'image_scan_memory_mb', type: 'integer', enum: ServerSetting::IMAGE_SCAN_MEMORY_OPTIONS_MB, description: 'Memory limit of each scanner step, in MB.'), new OA\Property(property: 'scanner_state', type: 'string', nullable: true, enum: ['pulling', 'updating_database', 'ready', 'failed'], description: 'State of the scanner on the server. Null when unknown.'), new OA\Property(property: 'scanner_failed_step', type: 'string', nullable: true, description: 'The step that failed when scanner_state is failed.'), new OA\Property(property: 'scanner_built_at', type: 'string', format: 'date-time', nullable: true, description: 'Time of the vulnerability database on the server.'), @@ -92,7 +96,7 @@ class ServerImageScanningController extends Controller #[OA\Patch( summary: 'Update image scanning settings', - description: 'Turn image vulnerability scanning of a server owned by the authenticated team on or off. Turning it on gets the scanner on the server in the background.', + description: 'Turn image vulnerability scanning of a server owned by the authenticated team on or off, or change the memory limit of the scanner. Send at least one field. Turning it on gets the scanner on the server in the background. A new memory limit applies to the next scan.', path: '/servers/{uuid}/image-scanning', operationId: 'update-server-image-scanning', security: [['bearerAuth' => []]], @@ -103,9 +107,9 @@ class ServerImageScanningController extends Controller requestBody: new OA\RequestBody( required: true, content: new OA\JsonContent( - required: ['is_image_scan_enabled'], properties: [ new OA\Property(property: 'is_image_scan_enabled', type: 'boolean'), + new OA\Property(property: 'image_scan_memory_mb', type: 'integer', enum: ServerSetting::IMAGE_SCAN_MEMORY_OPTIONS_MB, description: 'Memory limit of each scanner step, in MB.'), ], type: 'object', ), @@ -140,7 +144,8 @@ class ServerImageScanningController extends Controller $this->authorize('update', $server); $validator = customApiValidator($request->all(), [ - 'is_image_scan_enabled' => 'required|boolean', + 'is_image_scan_enabled' => 'required_without:image_scan_memory_mb|boolean', + 'image_scan_memory_mb' => ['required_without:is_image_scan_enabled', 'integer', Rule::in(ServerSetting::IMAGE_SCAN_MEMORY_OPTIONS_MB)], ]); $extraFields = array_diff(array_keys($request->all()), self::ALLOWED_FIELDS); if ($validator->fails() || ! empty($extraFields)) { @@ -155,19 +160,26 @@ class ServerImageScanningController extends Controller ], 422); } - $enabled = $request->boolean('is_image_scan_enabled'); + $audit = [ + 'team_id' => $teamId, + 'server_uuid' => $server->uuid, + 'server_name' => $server->name, + ]; try { - SetServerImageScanning::run($server, $enabled); + if ($request->has('image_scan_memory_mb')) { + SetServerImageScanning::ensureManagedHere($server); + $server->settings->update(['image_scan_memory_mb' => $request->integer('image_scan_memory_mb')]); + auditLog('api.server.image_scanner_memory_changed', $audit); + } + if ($request->has('is_image_scan_enabled')) { + $enabled = $request->boolean('is_image_scan_enabled'); + SetServerImageScanning::run($server, $enabled); + auditLog($enabled ? 'api.server.image_scanning_enabled' : 'api.server.image_scanning_disabled', $audit); + } } catch (ImageScanningRefused $e) { return response()->json(['message' => $e->getMessage()], 409); } - auditLog($enabled ? 'api.server.image_scanning_enabled' : 'api.server.image_scanning_disabled', [ - 'team_id' => $teamId, - 'server_uuid' => $server->uuid, - 'server_name' => $server->name, - ]); - return response()->json($this->transform($server->refresh())); } } diff --git a/app/Livewire/ImageScan/ServerImageScanning.php b/app/Livewire/ImageScan/ServerImageScanning.php index a2ab8f7cfe..b770ae7780 100644 --- a/app/Livewire/ImageScan/ServerImageScanning.php +++ b/app/Livewire/ImageScan/ServerImageScanning.php @@ -6,6 +6,7 @@ use App\Actions\ImageScan\ScanImage; use App\Actions\ImageScan\SetServerImageScanning; use App\Jobs\PrepareImageScannerJob; use App\Models\Server; +use App\Models\ServerSetting; use App\Services\ImageScan\ImageRef; use App\Services\ImageScan\ImageScanningRefused; use App\Services\ImageScan\ScannerCommands; @@ -14,6 +15,7 @@ use Carbon\CarbonImmutable; use Illuminate\Foundation\Auth\Access\AuthorizesRequests; use Illuminate\Support\Facades\Cache; use Illuminate\Support\Str; +use Illuminate\Validation\Rule; use Livewire\Component; class ServerImageScanning extends Component @@ -23,6 +25,8 @@ class ServerImageScanning extends Component public Server $server; + public string $scannerMemoryMb = ''; + public function getListeners() { return $this->teamChannelListeners([ @@ -34,6 +38,25 @@ class ServerImageScanning extends Component { $this->server = Server::ownedByCurrentTeam()->whereUuid($server_uuid)->firstOrFail(); $this->authorize('viewSecurity', $this->server); + $this->scannerMemoryMb = (string) $this->server->settings->image_scan_memory_mb; + } + + public function updatedScannerMemoryMb() + { + $this->authorize('update', $this->server); + + $this->validate(['scannerMemoryMb' => ['required', Rule::in(array_map('strval', ServerSetting::IMAGE_SCAN_MEMORY_OPTIONS_MB))]]); + + try { + SetServerImageScanning::ensureManagedHere($this->server); + $this->server->settings->update(['image_scan_memory_mb' => (int) $this->scannerMemoryMb]); + $this->auditImageScanning('image_scanner_memory_changed'); + $this->dispatch('success', 'Scanner memory limit saved. It applies to the next scan.'); + } catch (\Throwable $e) { + $this->scannerMemoryMb = (string) $this->server->settings->refresh()->image_scan_memory_mb; + + return handleError($e, $this); + } } public function scanUpdated(array $event = []): void @@ -135,6 +158,11 @@ class ServerImageScanning extends Component return view('livewire.image-scan.server-image-scanning', [ 'enabled' => (bool) $this->server->settings->is_image_scan_enabled, + 'memoryLimit' => formatBytes($this->server->settings->image_scan_memory_mb * 1024 * 1024, 0), + 'memoryOptions' => array_map( + fn (int $mb) => ['value' => (string) $mb, 'label' => formatBytes($mb * 1024 * 1024, 0)], + ServerSetting::IMAGE_SCAN_MEMORY_OPTIONS_MB, + ), 'prepareState' => is_string($state) ? $state : null, 'failedStep' => is_string($failedStep) ? $failedStep : null, 'registries' => collect(config('constants.image_scan.images')) diff --git a/app/Models/ServerSetting.php b/app/Models/ServerSetting.php index 9eb056de7b..7191ed2011 100644 --- a/app/Models/ServerSetting.php +++ b/app/Models/ServerSetting.php @@ -108,9 +108,16 @@ class ServerSetting extends Model */ public const string DEFAULT_AXIOM_DOMAIN = 'us-east-1.aws.edge.axiom.co'; + /** Memory limit of each scanner container (Syft, Grype, regctl), in MB. */ + public const int DEFAULT_IMAGE_SCAN_MEMORY_MB = 1024; + + /** @var list */ + public const array IMAGE_SCAN_MEMORY_OPTIONS_MB = [1024, 2048, 4096, 8192]; + protected $fillable = [ 'server_id', 'is_image_scan_enabled', + 'image_scan_memory_mb', 'image_candidate_scans_today', 'image_candidate_scans_date', 'is_swarm_manager', @@ -196,6 +203,7 @@ class ServerSetting extends Model protected $attributes = [ 'is_traffic_analytics_enabled' => false, 'is_image_scan_enabled' => false, + 'image_scan_memory_mb' => self::DEFAULT_IMAGE_SCAN_MEMORY_MB, 'image_candidate_scans_today' => 0, ]; @@ -230,6 +238,7 @@ class ServerSetting extends Model 'is_auto_security_updates_enabled' => 'boolean', 'is_logdrain_cloudwatch_enabled' => 'boolean', 'is_image_scan_enabled' => 'boolean', + 'image_scan_memory_mb' => 'integer', 'image_candidate_scans_today' => 'integer', 'image_candidate_scans_date' => 'date', 'ssh_host_key_trusted_at' => 'datetime', diff --git a/app/Services/ImageScan/ScannerCommands.php b/app/Services/ImageScan/ScannerCommands.php index 17c8aaaa1d..dd6af06852 100644 --- a/app/Services/ImageScan/ScannerCommands.php +++ b/app/Services/ImageScan/ScannerCommands.php @@ -3,6 +3,7 @@ namespace App\Services\ImageScan; use App\Models\Server; +use App\Models\ServerSetting; use App\Support\ValidationPatterns; use InvalidArgumentException; @@ -574,11 +575,14 @@ class ScannerCommands $env = array_merge([ 'SYFT_CHECK_FOR_APP_UPDATE' => 'false', // By default Syft runs NumCPU × 4 workers (`--cpus` does not lower NumCPU) and records metadata of every - // file a package owns. Together they need more than the 1 GB limit for an app image with many files + // file a package owns. Together they need more than the default 1 GB limit for an app image with many files // (Laravel with vendor and node_modules). Grype and the SBOM use only the packages, which stay the same. 'SYFT_PARALLELISM' => '1', 'SYFT_FILE_METADATA_SELECTION' => 'none', 'GRYPE_CHECK_FOR_APP_UPDATE' => 'false', + // Go collects garbage only when the heap doubles, so RSS can pass the container limit while the live + // data still fits. A soft limit below the container limit makes Go collect sooner instead of being killed. + 'GOMEMLIMIT' => intdiv($this->memoryMb() * 85, 100).'MiB', 'TMPDIR' => '/scratch', 'XDG_CACHE_HOME' => '/scratch/cache', ], $env); @@ -588,7 +592,7 @@ class ScannerCommands "timeout -k 10 {$timeout} docker run", '--name '.$this->containerName(), '--cap-drop ALL --security-opt no-new-privileges --read-only --pids-limit 256', - '--user '.self::SCANNER_USER.' --cpus 1 --memory 1g --memory-swap 1g --log-driver none', + '--user '.self::SCANNER_USER.' --cpus 1 --memory '.$this->memoryMb().'m --memory-swap '.$this->memoryMb().'m --log-driver none', ]; if (! $network) { $parts[] = '--network none'; @@ -779,6 +783,11 @@ class ScannerCommands return $this->shell($this->step('db-volume', 'docker volume create '.self::DB_VOLUME." >/dev/null && {$run} >/dev/null 2>&1")); } + private function memoryMb(): int + { + return (int) ($this->server->settings?->image_scan_memory_mb ?? ServerSetting::DEFAULT_IMAGE_SCAN_MEMORY_MB); + } + /** * @return array */ diff --git a/database/migrations/2026_11_09_130000_add_image_scan_memory_mb_to_server_settings_table.php b/database/migrations/2026_11_09_130000_add_image_scan_memory_mb_to_server_settings_table.php new file mode 100644 index 0000000000..d605b042e3 --- /dev/null +++ b/database/migrations/2026_11_09_130000_add_image_scan_memory_mb_to_server_settings_table.php @@ -0,0 +1,22 @@ +unsignedInteger('image_scan_memory_mb')->default(1024); + }); + } + + public function down(): void + { + Schema::table('server_settings', function (Blueprint $table) { + $table->dropColumn('image_scan_memory_mb'); + }); + } +}; diff --git a/openapi.json b/openapi.json index 0764e0e8ca..d823277d9a 100644 --- a/openapi.json +++ b/openapi.json @@ -19541,6 +19541,16 @@ "description": "Whether the server scans the images of its resources.", "type": "boolean" }, + "image_scan_memory_mb": { + "description": "Memory limit of each scanner step, in MB.", + "type": "integer", + "enum": [ + 1024, + 2048, + 4096, + 8192 + ] + }, "scanner_state": { "description": "State of the scanner on the server. Null when unknown.", "type": [ @@ -19606,7 +19616,7 @@ "Servers" ], "summary": "Update image scanning settings", - "description": "Turn image vulnerability scanning of a server owned by the authenticated team on or off. Turning it on gets the scanner on the server in the background.", + "description": "Turn image vulnerability scanning of a server owned by the authenticated team on or off, or change the memory limit of the scanner. Send at least one field. Turning it on gets the scanner on the server in the background. A new memory limit applies to the next scan.", "operationId": "update-server-image-scanning", "parameters": [ { @@ -19624,12 +19634,19 @@ "content": { "application\/json": { "schema": { - "required": [ - "is_image_scan_enabled" - ], "properties": { "is_image_scan_enabled": { "type": "boolean" + }, + "image_scan_memory_mb": { + "description": "Memory limit of each scanner step, in MB.", + "type": "integer", + "enum": [ + 1024, + 2048, + 4096, + 8192 + ] } }, "type": "object" @@ -29700,6 +29717,10 @@ "is_ssh_host_key_check_disabled": { "type": "boolean" }, + "is_ssh_host_key_check_enforced": { + "type": "boolean", + "description": "Refuse a changed host key on this server, also when the instance only warns." + }, "ssh_host_key_fingerprints": { "type": "array", "items": { diff --git a/openapi.yaml b/openapi.yaml index f1c7efdcac..873554e224 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -12421,6 +12421,7 @@ paths: schema: properties: is_image_scan_enabled: { description: 'Whether the server scans the images of its resources.', type: boolean } + image_scan_memory_mb: { description: 'Memory limit of each scanner step, in MB.', type: integer, enum: [1024, 2048, 4096, 8192] } scanner_state: { description: 'State of the scanner on the server. Null when unknown.', type: [string, 'null'], enum: [pulling, updating_database, ready, failed] } scanner_failed_step: { description: 'The step that failed when scanner_state is failed.', type: [string, 'null'] } scanner_built_at: { description: 'Time of the vulnerability database on the server.', type: [string, 'null'], format: date-time } @@ -12441,7 +12442,7 @@ paths: tags: - Servers summary: 'Update image scanning settings' - description: 'Turn image vulnerability scanning of a server owned by the authenticated team on or off. Turning it on gets the scanner on the server in the background.' + description: 'Turn image vulnerability scanning of a server owned by the authenticated team on or off, or change the memory limit of the scanner. Send at least one field. Turning it on gets the scanner on the server in the background. A new memory limit applies to the next scan.' operationId: update-server-image-scanning parameters: - @@ -12456,11 +12457,13 @@ paths: content: application/json: schema: - required: - - is_image_scan_enabled properties: is_image_scan_enabled: type: boolean + image_scan_memory_mb: + description: 'Memory limit of each scanner step, in MB.' + type: integer + enum: [1024, 2048, 4096, 8192] type: object responses: '200': @@ -18962,6 +18965,9 @@ components: description: 'Host key that the server sent when it did not match the pinned key.' is_ssh_host_key_check_disabled: type: boolean + is_ssh_host_key_check_enforced: + type: boolean + description: 'Refuse a changed host key on this server, also when the instance only warns.' ssh_host_key_fingerprints: type: array items: diff --git a/resources/views/livewire/image-scan/server-image-scanning.blade.php b/resources/views/livewire/image-scan/server-image-scanning.blade.php index 87aa2c1011..f21de79d1a 100644 --- a/resources/views/livewire/image-scan/server-image-scanning.blade.php +++ b/resources/views/livewire/image-scan/server-image-scanning.blade.php @@ -6,7 +6,7 @@ @php - $costText = 'Scans use up to 1 CPU and 1 GB of memory while they run, and about 1.2 GB of disk, plus temporary space for one image.'; + $costText = 'Scans use up to 1 CPU and ' . $memoryLimit . ' of memory while they run, and about 1.2 GB of disk, plus temporary space for one image.'; $pullState = match ($prepareState) { 'pulling' => 'running', 'updating_database', 'ready' => 'success', @@ -105,6 +105,12 @@
! $enabled || $prepareState !== 'ready', ])> +
+ +
diff --git a/tests/Feature/Api/ServerImageScanningApiTest.php b/tests/Feature/Api/ServerImageScanningApiTest.php new file mode 100644 index 0000000000..163191e512 --- /dev/null +++ b/tests/Feature/Api/ServerImageScanningApiTest.php @@ -0,0 +1,99 @@ + 'array', + 'session.driver' => 'array', + 'app.maintenance.driver' => 'file', + 'constants.ssh.mux_enabled' => false, + ]); + InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0], ['is_api_enabled' => true])); + + $this->team = Team::factory()->create(); + $this->user = User::factory()->create(); + $this->team->members()->attach($this->user->id, ['role' => 'owner']); + $this->server = Server::factory()->create([ + 'team_id' => $this->team->id, + 'private_key_id' => PrivateKey::factory()->create(['team_id' => $this->team->id])->id, + ]); +}); + +function imageScanningApiToken(User $user, Team $team, array $abilities): array +{ + $plainTextToken = Str::random(40); + $token = $user->tokens()->create([ + 'name' => 'image-scanning-api-test-'.Str::random(6), + 'token' => hash('sha256', $plainTextToken), + 'abilities' => $abilities, + 'team_id' => $team->id, + ]); + + return ['Authorization' => 'Bearer '.$token->getKey().'|'.$plainTextToken, 'Content-Type' => 'application/json']; +} + +it('returns the scanner memory limit of the server', function () { + $this->getJson("/api/v1/servers/{$this->server->uuid}/image-scanning", imageScanningApiToken($this->user, $this->team, ['read'])) + ->assertOk() + ->assertJsonPath('image_scan_memory_mb', 1024) + ->assertJsonPath('is_image_scan_enabled', false); +}); + +it('changes only the scanner memory limit', function () { + $this->patchJson("/api/v1/servers/{$this->server->uuid}/image-scanning", ['image_scan_memory_mb' => 4096], imageScanningApiToken($this->user, $this->team, ['write'])) + ->assertOk() + ->assertJsonPath('image_scan_memory_mb', 4096) + ->assertJsonPath('is_image_scan_enabled', false); + + expect($this->server->settings->fresh()->image_scan_memory_mb)->toBe(4096); + Queue::assertNothingPushed(); +}); + +it('refuses a memory limit that is not one of the options', function (array $body, string $field) { + $this->patchJson("/api/v1/servers/{$this->server->uuid}/image-scanning", $body, imageScanningApiToken($this->user, $this->team, ['write'])) + ->assertStatus(422) + ->assertJsonValidationErrors($field); + + expect($this->server->settings->fresh()->image_scan_memory_mb)->toBe(1024); +})->with([ + 'not an option' => [['image_scan_memory_mb' => 100], 'image_scan_memory_mb'], + 'not a number' => [['image_scan_memory_mb' => 'big'], 'image_scan_memory_mb'], + 'only unknown fields' => [['memory' => 2048], 'is_image_scan_enabled'], +]); + +it('does not change the memory limit of a server that another instance manages', function () { + $this->server->forceFill(['server_metadata' => ['transfer' => ['status' => 'transferred']]])->saveQuietly(); + + $this->patchJson("/api/v1/servers/{$this->server->uuid}/image-scanning", ['image_scan_memory_mb' => 2048], imageScanningApiToken($this->user, $this->team, ['write'])) + ->assertStatus(409); + + expect($this->server->settings->fresh()->image_scan_memory_mb)->toBe(1024); +}); + +it('does not let a member or another team change the memory limit', function () { + $member = User::factory()->create(); + $this->team->members()->attach($member->id, ['role' => 'member']); + $otherTeam = Team::factory()->create(); + $otherOwner = User::factory()->create(); + $otherTeam->members()->attach($otherOwner->id, ['role' => 'owner']); + + $this->patchJson("/api/v1/servers/{$this->server->uuid}/image-scanning", ['image_scan_memory_mb' => 8192], imageScanningApiToken($member, $this->team, ['write'])) + ->assertForbidden(); + app('auth')->forgetGuards(); + $this->patchJson("/api/v1/servers/{$this->server->uuid}/image-scanning", ['image_scan_memory_mb' => 8192], imageScanningApiToken($otherOwner, $otherTeam, ['write'])) + ->assertNotFound(); + + expect($this->server->settings->fresh()->image_scan_memory_mb)->toBe(1024); +}); diff --git a/tests/Feature/ImageScan/ScanImageTest.php b/tests/Feature/ImageScan/ScanImageTest.php index 4bf64c6288..0b16b5acae 100644 --- a/tests/Feature/ImageScan/ScanImageTest.php +++ b/tests/Feature/ImageScan/ScanImageTest.php @@ -497,7 +497,7 @@ it('says that the scanner ran out of memory only when Docker reports that the me expect($scan->status)->toBe(ImageScanStatus::Failed) ->and($scan->error_code)->toBe(ImageScanErrorCode::OutOfMemory) - ->and(ImageScanErrorCode::OutOfMemory->message())->toBe('The scanner ran out of memory and was stopped. It can use up to 1 GB, and the server must have that much free memory.'); + ->and(ImageScanErrorCode::OutOfMemory->message())->toBe('The scanner ran out of memory and was stopped. Raise the scanner memory limit on the Vulnerabilities page of the server, and make sure that the server has that much free memory.'); })->with([ 'syft' => ['sbom', "coolify-scan-exit save 0\ncoolify-scan-oom sbom\ncoolify-scan-exit sbom 137"], 'packages' => ['packages', "coolify-scan-oom packages\n".scanImageTestPayload('packages', '', 0, 137)], diff --git a/tests/Feature/ImageScan/ServerImageScanningTest.php b/tests/Feature/ImageScan/ServerImageScanningTest.php index 3407d82da3..da19a64a7e 100644 --- a/tests/Feature/ImageScan/ServerImageScanningTest.php +++ b/tests/Feature/ImageScan/ServerImageScanningTest.php @@ -215,6 +215,34 @@ describe('server image scanning page', function () { }); }); +describe('scanner memory limit', function () { + it('saves the scanner memory limit of the server and shows it in the cost text', function () { + imageScanningPage() + ->assertSet('scannerMemoryMb', '1024') + ->assertSee('1 CPU and 1 GB of memory') + ->set('scannerMemoryMb', '4096') + ->assertHasNoErrors() + ->assertDispatched('success') + ->assertSee('1 CPU and 4 GB of memory'); + + expect($this->server->settings->fresh()->image_scan_memory_mb)->toBe(4096); + }); + + it('refuses a memory limit that is not one of the options', function () { + imageScanningPage()->set('scannerMemoryMb', '100')->assertHasErrors('scannerMemoryMb'); + + expect($this->server->settings->fresh()->image_scan_memory_mb)->toBe(1024); + }); + + it('does not change the memory limit of a server that another instance manages', function () { + $this->server->forceFill(['server_metadata' => ['transfer' => ['status' => 'transferred']]])->saveQuietly(); + + imageScanningPage()->set('scannerMemoryMb', '2048')->assertNotDispatched('success'); + + expect($this->server->settings->fresh()->image_scan_memory_mb)->toBe(1024); + }); +}); + describe('server image scanning authorization', function () { beforeEach(function () { Queue::fake(); @@ -238,6 +266,7 @@ describe('server image scanning authorization', function () { $components = collect(['enable', 'disable', 'deleteScannerData']) ->mapWithKeys(fn (string $action) => [$action => Livewire::test(ServerImageScanning::class, ['server_uuid' => $this->server->uuid])]); + $memoryComponent = Livewire::test(ServerImageScanning::class, ['server_uuid' => $this->server->uuid]); $this->member->teams()->updateExistingPivot($this->team->id, ['role' => 'member']); $this->member->unsetRelation('teams'); @@ -246,8 +275,10 @@ describe('server image scanning authorization', function () { foreach ($components as $action => $component) { $component->call($action)->assertForbidden(); } + $memoryComponent->set('scannerMemoryMb', '8192')->assertForbidden(); - expect($this->server->settings->fresh()->is_image_scan_enabled)->toBeFalse(); + expect($this->server->settings->fresh()->is_image_scan_enabled)->toBeFalse() + ->and($this->server->settings->fresh()->image_scan_memory_mb)->toBe(1024); Queue::assertNothingPushed(); }); diff --git a/tests/Unit/ImageScan/ScannerCommandsTest.php b/tests/Unit/ImageScan/ScannerCommandsTest.php index aa9a37c4be..6f977c87dc 100644 --- a/tests/Unit/ImageScan/ScannerCommandsTest.php +++ b/tests/Unit/ImageScan/ScannerCommandsTest.php @@ -1,6 +1,7 @@ server = Mockery::mock(Server::class)->makePartial(); $this->server->shouldReceive('getAttribute')->with('user')->andReturn('ubuntu'); + $this->server->shouldReceive('getAttribute')->with('settings')->andReturn(new ServerSetting); $this->server->shouldReceive('setAttribute')->andReturnSelf(); $this->server->user = 'ubuntu'; $this->commands = new ScannerCommands($this->server, SCANNER_TEST_RUN_ID); @@ -87,7 +89,19 @@ it('runs syft with one worker and without file metadata, so a large image fits i expect($syft)->toContain("-e 'SYFT_PARALLELISM=1' ") ->and($syft)->toContain("-e 'SYFT_FILE_METADATA_SELECTION=none' ") - ->and($syft)->toContain('--memory 1g --memory-swap 1g'); + ->and($syft)->toContain('--memory 1024m --memory-swap 1024m'); +}); + +it('runs every scanner container with the memory limit of the server', function () { + $server = Mockery::mock(Server::class)->makePartial(); + $server->shouldReceive('getAttribute')->with('user')->andReturn('root'); + $server->shouldReceive('getAttribute')->with('settings')->andReturn(new ServerSetting(['image_scan_memory_mb' => 4096])); + $commands = new ScannerCommands($server, SCANNER_TEST_RUN_ID); + + $runs = collect(scannerCommandLists($commands))->flatMap(fn (array $list) => scannerDockerRuns($list)); + + expect($runs)->not->toBeEmpty() + ->and($runs->every(fn (string $line) => str_contains($line, '--memory 4096m --memory-swap 4096m ')))->toBeTrue(); }); it('asks Docker whether the memory limit stopped a scanner, then removes its container in the same script', function () { @@ -128,7 +142,7 @@ it('starts every command list by removing a leftover scan container', function ( it('runs every scanner container with each hardening flag exactly once', function () { $flags = ['--name coolify-scan-'.SCANNER_TEST_RUN_ID.' ', '--cap-drop ALL ', '--security-opt no-new-privileges ', '--read-only ', - '--pids-limit 256 ', '--user 65534:65534 ', '--cpus 1 ', '--memory 1g ', '--memory-swap 1g ', + '--pids-limit 256 ', '--user 65534:65534 ', '--cpus 1 ', '--memory 1024m ', '--memory-swap 1024m ', "-e 'SYFT_CHECK_FOR_APP_UPDATE=false' ", "-e 'GRYPE_CHECK_FOR_APP_UPDATE=false' ", "-e 'XDG_CACHE_HOME=/scratch/cache' "]; $runs = 0;