diff --git a/.ai/lessons.md b/.ai/lessons.md index d1e140e5eb..64dad24085 100644 --- a/.ai/lessons.md +++ b/.ai/lessons.md @@ -106,6 +106,11 @@ - The parsers add sudo only to line starts (and after `&&`, `||`, `|`, `$(`). Put `if`/`else` branches on their own lines. When a redirect or `cd` must stay, make the whole line one `sh -c '...'` script; it runs as `sudo sh -c` without inner sudo or bash. - Tests that replace `DatabaseStartCommandExecutor` or use root servers miss non-root bugs. Test the parsed commands of a non-root server. +## Redact sensitive output before every sink +- Use one fail-closed redaction policy for database logs, framework logs, notifications, API output, exports, and MCP responses. +- Replace known resource secrets before storage; format detection is only a second layer. +- Keep read-time redaction for legacy records, but do not use it as the primary control. + ## Cached and stored state - `StandaloneDocker`/`SwarmDocker` `server` uses `Server::findCached()`, a per-process identity map that is flushed only when the `servers` row updates. A `ServerSetting` save does not flush it, so pass the current `Server` instance (`setRelation('server', $server)`) when generated output depends on just-saved settings. - Application deployments use the stored `custom_labels` when they exist. When a server-level setting changes the generated labels, regenerate the stored labels of read-only (`is_container_label_readonly_enabled`) applications, or a redeploy keeps the old labels. diff --git a/app/Actions/CoolifyTask/RunRemoteProcess.php b/app/Actions/CoolifyTask/RunRemoteProcess.php index 90affa8db2..4f525ebcc8 100644 --- a/app/Actions/CoolifyTask/RunRemoteProcess.php +++ b/app/Actions/CoolifyTask/RunRemoteProcess.php @@ -6,19 +6,33 @@ use App\Enums\ActivityTypes; use App\Enums\ProcessStatus; use App\Helpers\SshMultiplexingHelper; use App\Jobs\ApplicationDeploymentJob; +use App\Models\Application; +use App\Models\EnvironmentVariable; use App\Models\Server; +use App\Models\Service; +use App\Models\ServiceApplication; +use App\Models\ServiceDatabase; +use App\Services\Security\SensitiveDataRedactor; use App\Support\DatabaseImport\DatabaseImportCleanup; use App\Support\RemoteProcessCommand; use App\Support\ResourceStartActivity; use App\Traits\BroadcastsToTeam; use Illuminate\Contracts\Process\ProcessResult; +use Illuminate\Process\Exceptions\ProcessTimedOutException; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Process; use Spatie\Activitylog\Models\Activity; +use Symfony\Component\Process\Exception\ProcessTimedOutException as SymfonyProcessTimedOutException; +use Throwable; class RunRemoteProcess { + /** + * A private key block that has no END line yet. + */ + private const UNFINISHED_PRIVATE_KEY_PATTERN = '/-----BEGIN [A-Z ]*PRIVATE KEY-----(?![\s\S]*-----END [A-Z ]*PRIVATE KEY-----)[\s\S]*/'; + public Activity $activity; public bool $hide_from_output; @@ -39,6 +53,21 @@ class RunRemoteProcess protected int $counter = 1; + /** + * Output of each stream that is not redacted yet, because its line or private key block is not complete. + * + * @var array + */ + private array $pendingOutput = []; + + /** + * Environment values of the activity's resource to hide in its output, resolved once. + * False when resolving them failed, so the output is fully redacted. + * + * @var array{known: list, locked: list}|false|null + */ + private array|false|null $logSecrets = null; + /** * Create a new job instance. */ @@ -83,16 +112,23 @@ class RunRemoteProcess $status = ProcessStatus::IN_PROGRESS; $timeout = config('constants.ssh.command_timeout'); - $process = Process::timeout($timeout)->start($this->getCommand(), $this->handleOutput(...)); - // Start from the stored properties and save now, so later output writes only change the log and - // never write back an old copy over a status that Coolify set meanwhile (for example a stopped import). - $stored = Activity::query()->whereKey($this->activity->getKey())->first()?->properties ?? $this->activity->properties; - $this->activity->properties = $stored->merge([ - 'process_id' => $process->id(), - ]); - $this->activity->save(); + try { + $process = Process::timeout($timeout)->start($this->getCommand(), $this->handleOutput(...)); + // Start from the stored properties and save now, so later output writes only change the log and + // never write back an old copy over a status that Coolify set meanwhile (for example a stopped import). + $stored = Activity::query()->whereKey($this->activity->getKey())->first()?->properties ?? $this->activity->properties; + $this->activity->properties = $stored->merge([ + 'process_id' => $process->id(), + ]); + $this->activity->save(); - $processResult = $process->wait(); + $processResult = $process->wait(); + } catch (ProcessTimedOutException|SymfonyProcessTimedOutException) { + // The timeout message contains the full SSH command, which can contain secrets. + throw new \RuntimeException("The command on the server timed out after {$timeout} seconds.", 124); + } finally { + $this->flushPendingOutput(); + } // A database import whose SSH command ended without the restore result keeps blocking the // database until its restore is stopped. The stop event also cleans up, so the normal @@ -103,13 +139,13 @@ class RunRemoteProcess && DatabaseImportCleanup::stopAfterTransportFailure($this->activity, $processResult->exitCode())) { $this->activity->properties = $this->activity->properties->merge([ 'exitCode' => $processResult->exitCode(), - 'stdout' => $processResult->output(), - 'stderr' => $processResult->errorOutput(), + 'stdout' => $this->redact($processResult->output()), + 'stderr' => $this->redact($processResult->errorOutput()), ]); $this->activity->save(); if (! $this->ignore_errors) { - throw new \RuntimeException($processResult->errorOutput(), $processResult->exitCode()); + throw new \RuntimeException($this->redact($processResult->errorOutput()), $processResult->exitCode()); } return $processResult; @@ -127,8 +163,8 @@ class RunRemoteProcess $properties = [ 'exitCode' => $processResult->exitCode(), - 'stdout' => $processResult->output(), - 'stderr' => $processResult->errorOutput(), + 'stdout' => $this->redact($processResult->output()), + 'stderr' => $this->redact($processResult->errorOutput()), 'status' => $status->value, ]; @@ -148,12 +184,12 @@ class RunRemoteProcess if ($this->call_event_on_finish) { try { self::dispatchFinishEvent($this->activity, $this->call_event_on_finish, $this->call_event_data); - } catch (\Throwable $e) { + } catch (Throwable $e) { Log::error('Error calling event: '.$e->getMessage()); } } if ($processResult->exitCode() != 0 && ! $this->ignore_errors) { - throw new \RuntimeException($processResult->errorOutput(), $processResult->exitCode()); + throw new \RuntimeException($this->redact($processResult->errorOutput()), $processResult->exitCode()); } return $processResult; @@ -192,6 +228,11 @@ class RunRemoteProcess if ($this->hide_from_output) { return; } + $this->pendingOutput[$type] = ($this->pendingOutput[$type] ?? '').$output; + $output = self::takeCompleteOutput($this->pendingOutput[$type]); + if ($output === '') { + return; + } $this->current_time = $this->elapsedTime(); $this->activity->description = $this->encodeOutput($type, $output); if ($this->isAfterLastThrottle()) { @@ -203,6 +244,51 @@ class RunRemoteProcess } } + /** + * Write the output that is still buffered when the process ends, also after a timeout or failure. + */ + private function flushPendingOutput(): void + { + $written = false; + foreach ($this->pendingOutput as $type => $pending) { + $output = self::takeCompleteOutput($this->pendingOutput[$type], final: true); + if ($output !== '') { + $this->activity->description = $this->encodeOutput($type, $output); + $written = true; + } + } + if ($written) { + $this->activity->save(); + } + } + + /** + * Take the complete lines from the buffered output of a stream, so a secret split across output + * chunks is redacted as a whole. A private key block stays buffered until its END line. With + * $final, all buffered output is taken, and an unfinished private key block is hidden. + */ + public static function takeCompleteOutput(string &$pending, bool $final = false): string + { + if ($final) { + $output = preg_replace(self::UNFINISHED_PRIVATE_KEY_PATTERN, REDACTED, $pending) ?? REDACTED; + $pending = ''; + + return $output; + } + + $end = max(strrpos($pending, "\n"), strrpos($pending, "\r")); + if ($end === false) { + return ''; + } + $output = substr($pending, 0, $end + 1); + if (preg_match(self::UNFINISHED_PRIVATE_KEY_PATTERN, $output, $match, PREG_OFFSET_CAPTURE) === 1) { + $output = substr($output, 0, $match[0][1]); + } + $pending = substr($pending, strlen($output)); + + return $output; + } + protected function elapsedTime(): int { $timeMs = (hrtime(true) - $this->time_start) / 1_000_000; @@ -215,7 +301,7 @@ class RunRemoteProcess $outputStack = json_decode($this->activity->description, associative: true, flags: JSON_THROW_ON_ERROR | JSON_UNESCAPED_UNICODE); $outputStack[] = [ 'type' => $type, - 'output' => $output, + 'output' => $this->redact($output), 'timestamp' => hrtime(true), 'batch' => ApplicationDeploymentJob::$batch_counter, 'order' => $this->getLatestCounter(), @@ -234,6 +320,50 @@ class RunRemoteProcess return end($description)['order'] + 1; } + private function redact(string $value): string + { + $this->logSecrets ??= $this->resolveLogSecrets(); + if ($this->logSecrets === false) { + return REDACTED; + } + + return resolve(SensitiveDataRedactor::class)->redactText($value, $this->logSecrets['known'], $this->logSecrets['locked']); + } + + /** + * The environment values of the application or service this activity runs for, following + * the resource's log redaction setting. + * + * @return array{known: list, locked: list}|false + */ + private function resolveLogSecrets(): array|false + { + try { + return EnvironmentVariable::logRedactionSecretsFor($this->logResource()); + } catch (Throwable) { + return false; + } + } + + private function logResource(): Application|Service|null + { + $subject = $this->activity->subject_type !== null ? $this->activity->subject : null; + if ($subject instanceof ServiceApplication || $subject instanceof ServiceDatabase) { + return $subject->service; + } + if ($subject instanceof Application || $subject instanceof Service) { + return $subject; + } + + $typeUuid = $this->activity->getExtraProperty('type_uuid'); + if (! is_string($typeUuid) || $typeUuid === '') { + return null; + } + + return Service::query()->where('uuid', $typeUuid)->first() + ?? Application::query()->where('uuid', $typeUuid)->first(); + } + /** * Determines if it's time to write again to database. * diff --git a/app/Actions/Server/CleanupDocker.php b/app/Actions/Server/CleanupDocker.php index 888169be5b..c49c057060 100644 --- a/app/Actions/Server/CleanupDocker.php +++ b/app/Actions/Server/CleanupDocker.php @@ -180,7 +180,7 @@ class CleanupDocker implements ShouldBeUnique if ($commandOutput !== null) { $cleanupLog[] = [ 'command' => $command, - 'output' => $commandOutput, + 'output' => remove_iip($commandOutput), ]; } } @@ -363,7 +363,7 @@ class CleanupDocker implements ShouldBeUnique $deleteOutput = $this->runRemoteCommand($deleteCommand, $server); $cleanupLog[] = [ 'command' => $deleteCommand, - 'output' => $deleteOutput ?? 'Images removed or were in use', + 'output' => remove_iip($deleteOutput ?? 'Images removed or were in use'), ]; } diff --git a/app/Actions/Server/DeleteServer.php b/app/Actions/Server/DeleteServer.php index 428d696fdc..bcffee2a50 100644 --- a/app/Actions/Server/DeleteServer.php +++ b/app/Actions/Server/DeleteServer.php @@ -108,7 +108,7 @@ class DeleteServer // Notify the team about the failure $team = Team::find($teamId); - $team?->notify(new HetznerDeletionFailed($hetznerServerId, $teamId, $e->getMessage())); + $team?->notify(new HetznerDeletionFailed($hetznerServerId, $teamId, remove_iip($e->getMessage()))); } } diff --git a/app/Http/Controllers/Api/ApplicationsController.php b/app/Http/Controllers/Api/ApplicationsController.php index 9f3110b72f..4096ef94e5 100644 --- a/app/Http/Controllers/Api/ApplicationsController.php +++ b/app/Http/Controllers/Api/ApplicationsController.php @@ -50,6 +50,7 @@ class ApplicationsController extends Controller 'inject_build_args_to_dockerfile', 'include_source_commit_in_build', 'is_env_sorting_enabled', + 'redact_all_env_values_in_logs', 'is_pr_deployments_public_enabled', 'stop_grace_period', 'docker_images_to_keep', @@ -76,6 +77,7 @@ class ApplicationsController extends Controller 'inject_build_args_to_dockerfile', 'include_source_commit_in_build', 'is_env_sorting_enabled', + 'redact_all_env_values_in_logs', 'is_pr_deployments_public_enabled', 'is_gzip_enabled', 'is_stripprefix_enabled', @@ -429,6 +431,7 @@ class ApplicationsController extends Controller 'inject_build_args_to_dockerfile' => ['type' => 'boolean', 'description' => 'Inject build arguments into the Dockerfile build.'], 'include_source_commit_in_build' => ['type' => 'boolean', 'description' => 'Include the source commit in the build.'], 'is_env_sorting_enabled' => ['type' => 'boolean', 'description' => 'Sort environment variables.'], + 'redact_all_env_values_in_logs' => ['type' => 'boolean', 'description' => 'Redact all environment variable values in logs. When false, only locked (shown once) values are redacted. Requires team admin permissions.'], 'is_pr_deployments_public_enabled' => ['type' => 'boolean', 'description' => 'Make pull request deployments public.'], 'stop_grace_period' => ['type' => 'integer', 'nullable' => true, 'minimum' => 1, 'maximum' => 3600, 'description' => 'Container stop grace period in seconds.'], 'docker_images_to_keep' => ['type' => 'integer', 'minimum' => 0, 'maximum' => 100, 'description' => 'Number of Docker images to retain.'], @@ -629,6 +632,7 @@ class ApplicationsController extends Controller 'inject_build_args_to_dockerfile' => ['type' => 'boolean', 'description' => 'Inject build arguments into the Dockerfile build.'], 'include_source_commit_in_build' => ['type' => 'boolean', 'description' => 'Include the source commit in the build.'], 'is_env_sorting_enabled' => ['type' => 'boolean', 'description' => 'Sort environment variables.'], + 'redact_all_env_values_in_logs' => ['type' => 'boolean', 'description' => 'Redact all environment variable values in logs. When false, only locked (shown once) values are redacted. Requires team admin permissions.'], 'is_pr_deployments_public_enabled' => ['type' => 'boolean', 'description' => 'Make pull request deployments public.'], 'stop_grace_period' => ['type' => 'integer', 'nullable' => true, 'minimum' => 1, 'maximum' => 3600, 'description' => 'Container stop grace period in seconds.'], 'docker_images_to_keep' => ['type' => 'integer', 'minimum' => 0, 'maximum' => 100, 'description' => 'Number of Docker images to retain.'], @@ -829,6 +833,7 @@ class ApplicationsController extends Controller 'inject_build_args_to_dockerfile' => ['type' => 'boolean', 'description' => 'Inject build arguments into the Dockerfile build.'], 'include_source_commit_in_build' => ['type' => 'boolean', 'description' => 'Include the source commit in the build.'], 'is_env_sorting_enabled' => ['type' => 'boolean', 'description' => 'Sort environment variables.'], + 'redact_all_env_values_in_logs' => ['type' => 'boolean', 'description' => 'Redact all environment variable values in logs. When false, only locked (shown once) values are redacted. Requires team admin permissions.'], 'is_pr_deployments_public_enabled' => ['type' => 'boolean', 'description' => 'Make pull request deployments public.'], 'stop_grace_period' => ['type' => 'integer', 'nullable' => true, 'minimum' => 1, 'maximum' => 3600, 'description' => 'Container stop grace period in seconds.'], 'docker_images_to_keep' => ['type' => 'integer', 'minimum' => 0, 'maximum' => 100, 'description' => 'Number of Docker images to retain.'], @@ -1000,6 +1005,7 @@ class ApplicationsController extends Controller 'inject_build_args_to_dockerfile' => ['type' => 'boolean', 'description' => 'Inject build arguments into the Dockerfile build.'], 'include_source_commit_in_build' => ['type' => 'boolean', 'description' => 'Include the source commit in the build.'], 'is_env_sorting_enabled' => ['type' => 'boolean', 'description' => 'Sort environment variables.'], + 'redact_all_env_values_in_logs' => ['type' => 'boolean', 'description' => 'Redact all environment variable values in logs. When false, only locked (shown once) values are redacted. Requires team admin permissions.'], 'is_pr_deployments_public_enabled' => ['type' => 'boolean', 'description' => 'Make pull request deployments public.'], 'stop_grace_period' => ['type' => 'integer', 'nullable' => true, 'minimum' => 1, 'maximum' => 3600, 'description' => 'Container stop grace period in seconds.'], 'docker_images_to_keep' => ['type' => 'integer', 'minimum' => 0, 'maximum' => 100, 'description' => 'Number of Docker images to retain.'], @@ -1167,6 +1173,7 @@ class ApplicationsController extends Controller 'inject_build_args_to_dockerfile' => ['type' => 'boolean', 'description' => 'Inject build arguments into the Dockerfile build.'], 'include_source_commit_in_build' => ['type' => 'boolean', 'description' => 'Include the source commit in the build.'], 'is_env_sorting_enabled' => ['type' => 'boolean', 'description' => 'Sort environment variables.'], + 'redact_all_env_values_in_logs' => ['type' => 'boolean', 'description' => 'Redact all environment variable values in logs. When false, only locked (shown once) values are redacted. Requires team admin permissions.'], 'is_pr_deployments_public_enabled' => ['type' => 'boolean', 'description' => 'Make pull request deployments public.'], 'stop_grace_period' => ['type' => 'integer', 'nullable' => true, 'minimum' => 1, 'maximum' => 3600, 'description' => 'Container stop grace period in seconds.'], 'docker_images_to_keep' => ['type' => 'integer', 'minimum' => 0, 'maximum' => 100, 'description' => 'Number of Docker images to retain.'], @@ -2700,7 +2707,10 @@ class ApplicationsController extends Controller $lines = normalizeLogLines($request->query('lines')); $showTimestamps = parseLogTimestampFlag($request->query('show_timestamps')); - $logs = getContainerLogs($application->destination->server, $container['ID'], $lines, $showTimestamps); + $logs = sanitizeLogsForExport( + getContainerLogs($application->destination->server, $container['ID'], $lines, $showTimestamps), + EnvironmentVariable::logRedactionSecretsFor($application, includePreview: $pullRequestId !== null), + ); return response()->json([ 'logs' => $logs, @@ -3172,6 +3182,7 @@ class ApplicationsController extends Controller 'inject_build_args_to_dockerfile' => ['type' => 'boolean', 'description' => 'Inject build arguments into the Dockerfile build.'], 'include_source_commit_in_build' => ['type' => 'boolean', 'description' => 'Include the source commit in the build.'], 'is_env_sorting_enabled' => ['type' => 'boolean', 'description' => 'Sort environment variables.'], + 'redact_all_env_values_in_logs' => ['type' => 'boolean', 'description' => 'Redact all environment variable values in logs. When false, only locked (shown once) values are redacted. Requires team admin permissions.'], 'is_pr_deployments_public_enabled' => ['type' => 'boolean', 'description' => 'Make pull request deployments public.'], 'stop_grace_period' => ['type' => 'integer', 'nullable' => true, 'minimum' => 1, 'maximum' => 3600, 'description' => 'Container stop grace period in seconds.'], 'docker_images_to_keep' => ['type' => 'integer', 'minimum' => 0, 'maximum' => 100, 'description' => 'Number of Docker images to retain.'], @@ -3278,6 +3289,10 @@ class ApplicationsController extends Controller $this->authorize('update', $application); + if ($request->has('redact_all_env_values_in_logs')) { + $this->authorize('manageEnvironment', $application); + } + if ($request->instant_deploy) { abort_unless($request->user()->tokenCan('deploy') || $request->user()->tokenCan('root'), 403, 'Missing required permissions: deploy'); $this->authorize('deploy', $application); diff --git a/app/Http/Controllers/Api/DatabasesController.php b/app/Http/Controllers/Api/DatabasesController.php index b6fe02d8bc..fd7c53e90a 100644 --- a/app/Http/Controllers/Api/DatabasesController.php +++ b/app/Http/Controllers/Api/DatabasesController.php @@ -18,6 +18,7 @@ use App\Models\LocalPersistentVolume; use App\Models\Project; use App\Models\S3Storage; use App\Models\ScheduledDatabaseBackup; +use App\Models\ScheduledDatabaseBackupExecution; use App\Models\Server; use App\Models\StandaloneDocker; use App\Models\StandaloneDragonfly; @@ -355,11 +356,19 @@ class DatabasesController extends Controller } $databaseIds = $databases->pluck('id')->toArray(); + $databasesByMorphKey = $databases->keyBy(fn ($database) => $database->getMorphClass().':'.$database->id); + $messageSecrets = []; $backupConfigs = ScheduledDatabaseBackup::ownedByCurrentTeamAPI($teamId)->with(['latest_log', 's3', 's3Storages']) ->whereIn('database_id', $databaseIds) ->get() - ->map(fn (ScheduledDatabaseBackup $backup) => $this->withBackupS3StorageUuids($backup)) + ->map(function (ScheduledDatabaseBackup $backup) use ($databasesByMorphKey, &$messageSecrets): ScheduledDatabaseBackup { + if ($backup->latest_log) { + $this->redactBackupExecutionMessage($backup->latest_log, $databasesByMorphKey->get($backup->database_type.':'.$backup->database_id), $messageSecrets); + } + + return $this->withBackupS3StorageUuids($backup); + }) ->groupBy(fn (ScheduledDatabaseBackup $backup) => $backup->database_type.':'.$backup->database_id); $databases = $databases->map(function ($database) use ($backupConfigs) { @@ -431,8 +440,15 @@ class DatabasesController extends Controller $this->authorize('view', $database); + $messageSecrets = []; $backupConfig = ScheduledDatabaseBackup::ownedByCurrentTeamAPI($teamId)->with(['executions', 's3', 's3Storages'])->where('database_id', $database->id)->get() - ->map(fn (ScheduledDatabaseBackup $backup) => $this->withBackupS3StorageUuids($backup)); + ->map(function (ScheduledDatabaseBackup $backup) use ($database, &$messageSecrets): ScheduledDatabaseBackup { + $backup->executions->each(function (ScheduledDatabaseBackupExecution $execution) use ($database, &$messageSecrets): void { + $this->redactBackupExecutionMessage($execution, $database, $messageSecrets); + }); + + return $this->withBackupS3StorageUuids($backup); + }); return response()->json($backupConfig); } @@ -573,6 +589,7 @@ class DatabasesController extends Controller 'health_check_timeout' => ['type' => 'integer', 'description' => 'Healthcheck timeout in seconds.', 'minimum' => 1, 'default' => 5], 'health_check_retries' => ['type' => 'integer', 'description' => 'Healthcheck retries count.', 'minimum' => 1, 'default' => 5], 'health_check_start_period' => ['type' => 'integer', 'description' => 'Healthcheck start period in seconds.', 'minimum' => 0, 'default' => 5], + 'redact_all_env_values_in_logs' => ['type' => 'boolean', 'description' => 'Redact all environment variable values in logs. When false, only locked (shown once) values, database passwords, and remote secrets are redacted. Requires team admin permissions.'], ], ), ) @@ -627,6 +644,7 @@ class DatabasesController extends Controller 'limits_cpus' => 'string', 'limits_cpuset' => 'string|nullable', 'limits_cpu_shares' => 'numeric', + 'redact_all_env_values_in_logs' => 'boolean', ]); if ($validator->fails()) { @@ -636,6 +654,7 @@ class DatabasesController extends Controller ], 422); } $uuid = $request->uuid; + $redactAllEnvValuesInLogs = $request->has('redact_all_env_values_in_logs') ? $request->boolean('redact_all_env_values_in_logs') : null; removeUnnecessaryFieldsFromRequest($request); $database = queryDatabaseByUuidWithinTeam($uuid, $teamId); if (! $database) { @@ -644,6 +663,10 @@ class DatabasesController extends Controller $this->authorize('update', $database); + if ($redactAllEnvValuesInLogs !== null) { + $this->authorize('manageEnvironment', $database); + } + if ($request->is_public && $request->public_port) { if (isPublicPortAlreadyUsed($database->destination->server, $request->public_port, $database->id)) { return response()->json(['message' => 'Public port already used by another database.'], 400); @@ -876,6 +899,9 @@ class DatabasesController extends Controller } // Only update database fields, not backup configuration + if ($redactAllEnvValuesInLogs !== null) { + $database->redact_all_env_values_in_logs = $redactAllEnvValuesInLogs; + } $database->update($request->only($allowedFields)); if ($whatToDoWithDatabaseProxy === 'start') { @@ -889,7 +915,10 @@ class DatabasesController extends Controller 'database_uuid' => $database->uuid, 'database_name' => $database->name, 'database_type' => $database->type(), - 'changed_fields' => array_values(array_intersect($allowedFields, array_keys($request->all()))), + 'changed_fields' => [ + ...array_values(array_intersect($allowedFields, array_keys($request->all()))), + ...($redactAllEnvValuesInLogs === null ? [] : ['redact_all_env_values_in_logs']), + ], ]); return response()->json([ @@ -2783,7 +2812,10 @@ class DatabasesController extends Controller $lines = normalizeLogLines($request->query('lines')); $showTimestamps = parseLogTimestampFlag($request->query('show_timestamps')); - $logs = getContainerLogs($database->destination->server, $container['ID'], $lines, $showTimestamps); + $logs = sanitizeLogsForExport( + getContainerLogs($database->destination->server, $container['ID'], $lines, $showTimestamps), + EnvironmentVariable::logRedactionSecretsFor($database), + ); return response()->json([ 'logs' => $logs, @@ -3145,7 +3177,7 @@ class DatabasesController extends Controller #[OA\Get( summary: 'List backup executions', - description: 'Get all executions for a specific backup configuration.', + description: 'Get all executions for a specific backup configuration. The execution `message` is only returned (redacted) to tokens with the `read:sensitive` or `root` ability; otherwise it is `null`.', path: '/databases/{uuid}/backups/{scheduled_backup_uuid}/executions', operationId: 'list-backup-executions', security: [ @@ -3185,7 +3217,7 @@ class DatabasesController extends Controller new OA\Property(property: 'filename', type: 'string'), new OA\Property(property: 'size', type: 'integer'), new OA\Property(property: 'created_at', type: 'string'), - new OA\Property(property: 'message', type: 'string'), + new OA\Property(property: 'message', type: 'string', nullable: true, description: 'Redacted execution output. Null unless the token has the `read:sensitive` or `root` ability.'), new OA\Property(property: 'status', type: 'string'), ] ) @@ -3226,10 +3258,13 @@ class DatabasesController extends Controller } // Get all executions for this backup configuration + $messageSecrets = []; $executions = $backup->executions() ->orderBy('created_at', 'desc') ->get() - ->map(function ($execution) { + ->map(function ($execution) use ($database, &$messageSecrets) { + $this->redactBackupExecutionMessage($execution, $database, $messageSecrets); + return [ 'uuid' => $execution->uuid, 'filename' => $execution->filename, @@ -5282,4 +5317,34 @@ class DatabasesController extends Controller 'message' => 'Database cloned.', ], 201); } + + /** + * Execution output can embed credentials: hide it unless the token can read sensitive data, and redact it when shown. + * The database secrets are loaded once per database and kept in $secretsByDatabase for the rest of the request. + * + * @param array, locked: array}|null> $secretsByDatabase + */ + private function redactBackupExecutionMessage(ScheduledDatabaseBackupExecution $execution, ?Model $database, array &$secretsByDatabase): void + { + $message = $execution->message; + if (request()->attributes->get('can_read_sensitive', false) !== true || ! is_string($message) || ! $database) { + $execution->setAttribute('message', null); + + return; + } + + $key = $database->getMorphClass().':'.$database->getKey(); + if (! array_key_exists($key, $secretsByDatabase)) { + try { + // Use a copy so the relations loaded for redaction do not end up in the response. + $secretsByDatabase[$key] = EnvironmentVariable::logRedactionSecretsFor(clone $database); + } catch (\Throwable $e) { + report($e); + $secretsByDatabase[$key] = null; + } + } + + // Without the database secrets the message cannot be redacted reliably, so hide it. + $execution->setAttribute('message', $secretsByDatabase[$key] === null ? null : sanitizeLogsForExport($message, $secretsByDatabase[$key])); + } } diff --git a/app/Http/Controllers/Api/ServiceApplicationsController.php b/app/Http/Controllers/Api/ServiceApplicationsController.php index e97e5e42a9..30cd61f0b9 100644 --- a/app/Http/Controllers/Api/ServiceApplicationsController.php +++ b/app/Http/Controllers/Api/ServiceApplicationsController.php @@ -7,6 +7,7 @@ use App\Actions\Service\RestartServiceApplication; use App\Actions\Service\StopServiceApplication; use App\Actions\Service\UpdateServiceApplicationFromApi; use App\Http\Controllers\Controller; +use App\Models\EnvironmentVariable; use App\Models\ServerSetting; use App\Models\Service; use App\Models\ServiceApplication; @@ -532,7 +533,10 @@ class ServiceApplicationsController extends Controller } $lines = normalizeLogLines($request->query('lines')); - $logs = getContainerLogs($server, $containerName, $lines); + $logs = sanitizeLogsForExport( + getContainerLogs($server, $containerName, $lines), + EnvironmentVariable::logRedactionSecretsFor($serviceApplication->service), + ); return response()->json([ 'logs' => $logs, diff --git a/app/Http/Controllers/Api/ServiceDatabasesController.php b/app/Http/Controllers/Api/ServiceDatabasesController.php index 795778ffb0..ba0e7c9045 100644 --- a/app/Http/Controllers/Api/ServiceDatabasesController.php +++ b/app/Http/Controllers/Api/ServiceDatabasesController.php @@ -8,6 +8,7 @@ use App\Actions\Service\DeployServiceApplication; use App\Actions\Service\RestartServiceApplication; use App\Actions\Service\StopServiceApplication; use App\Http\Controllers\Controller; +use App\Models\EnvironmentVariable; use App\Models\Service; use App\Models\ServiceDatabase; use Illuminate\Http\JsonResponse; @@ -401,7 +402,10 @@ class ServiceDatabasesController extends Controller $lines = normalizeLogLines($request->query('lines')); return response()->json([ - 'logs' => getContainerLogs($server, $containerName, $lines), + 'logs' => sanitizeLogsForExport( + getContainerLogs($server, $containerName, $lines), + EnvironmentVariable::logRedactionSecretsFor($serviceDatabase->service), + ), ]); } diff --git a/app/Http/Controllers/Api/ServicesController.php b/app/Http/Controllers/Api/ServicesController.php index 01f3657434..c858c1bc8c 100644 --- a/app/Http/Controllers/Api/ServicesController.php +++ b/app/Http/Controllers/Api/ServicesController.php @@ -321,6 +321,7 @@ class ServicesController extends Controller ], 'force_domain_override' => ['type' => 'boolean', 'default' => false, 'description' => 'Force domain override even if conflicts are detected.'], 'is_container_label_escape_enabled' => ['type' => 'boolean', 'default' => true, 'description' => 'Escape special characters in labels. By default, $ (and other chars) is escaped. If you want to use env variables inside the labels, turn this off.'], + 'redact_all_env_values_in_logs' => ['type' => 'boolean', 'default' => true, 'description' => 'Redact all environment variable values in logs. When false, only locked (shown once) values are redacted. Requires team admin permissions.'], 'tags' => ['type' => 'array', 'items' => new OA\Items(type: 'string'), 'description' => 'Tags to assign to the service.'], ], ), @@ -388,7 +389,7 @@ class ServicesController extends Controller )] public function create_service(Request $request) { - $allowedFields = ['type', 'name', 'description', 'project_uuid', 'environment_name', 'environment_uuid', 'server_uuid', 'destination_uuid', 'instant_deploy', 'docker_compose_raw', 'urls', 'force_domain_override', 'is_container_label_escape_enabled', 'tags']; + $allowedFields = ['type', 'name', 'description', 'project_uuid', 'environment_name', 'environment_uuid', 'server_uuid', 'destination_uuid', 'instant_deploy', 'docker_compose_raw', 'urls', 'force_domain_override', 'is_container_label_escape_enabled', 'redact_all_env_values_in_logs', 'tags']; $teamId = getTeamIdFromToken(); if (is_null($teamId)) { @@ -422,6 +423,7 @@ class ServicesController extends Controller 'urls.*.url' => ValidationPatterns::applicationDomainRules(), 'force_domain_override' => 'boolean', 'is_container_label_escape_enabled' => 'boolean', + 'redact_all_env_values_in_logs' => 'boolean', 'tags' => 'array|nullable', 'tags.*' => 'string|min:2', ]; @@ -554,6 +556,9 @@ class ServicesController extends Controller if ($request->has('is_container_label_escape_enabled')) { $service->is_container_label_escape_enabled = $request->boolean('is_container_label_escape_enabled'); } + if ($request->has('redact_all_env_values_in_logs')) { + $service->redact_all_env_values_in_logs = $request->boolean('redact_all_env_values_in_logs'); + } $service->save(); if ($oneClickDotEnvs?->count() > 0) { $oneClickDotEnvs->each(function ($value) use ($service) { @@ -622,7 +627,7 @@ class ServicesController extends Controller return response()->json(['message' => 'Service not found.', 'valid_service_types' => $serviceKeys], 404); } elseif (filled($request->docker_compose_raw)) { - $allowedFields = ['name', 'description', 'project_uuid', 'environment_name', 'environment_uuid', 'server_uuid', 'destination_uuid', 'instant_deploy', 'docker_compose_raw', 'connect_to_docker_network', 'urls', 'force_domain_override', 'is_container_label_escape_enabled', 'tags']; + $allowedFields = ['name', 'description', 'project_uuid', 'environment_name', 'environment_uuid', 'server_uuid', 'destination_uuid', 'instant_deploy', 'docker_compose_raw', 'connect_to_docker_network', 'urls', 'force_domain_override', 'is_container_label_escape_enabled', 'redact_all_env_values_in_logs', 'tags']; $validationRules = [ 'project_uuid' => 'string|required', @@ -641,6 +646,7 @@ class ServicesController extends Controller 'urls.*.url' => ValidationPatterns::applicationDomainRules(), 'force_domain_override' => 'boolean', 'is_container_label_escape_enabled' => 'boolean', + 'redact_all_env_values_in_logs' => 'boolean', 'tags' => 'array|nullable', 'tags.*' => 'string|min:2', ]; @@ -757,6 +763,9 @@ class ServicesController extends Controller if ($request->has('is_container_label_escape_enabled')) { $service->is_container_label_escape_enabled = $request->boolean('is_container_label_escape_enabled'); } + if ($request->has('redact_all_env_values_in_logs')) { + $service->redact_all_env_values_in_logs = $request->boolean('redact_all_env_values_in_logs'); + } $service->save(); $service->parse(isNew: true); @@ -985,7 +994,10 @@ class ServicesController extends Controller $lines = normalizeLogLines($request->query('lines')); $showTimestamps = parseLogTimestampFlag($request->query('show_timestamps')); - $logs = getContainerLogs($service->destination->server, $container['ID'], $lines, $showTimestamps); + $logs = sanitizeLogsForExport( + getContainerLogs($service->destination->server, $container['ID'], $lines, $showTimestamps), + EnvironmentVariable::logRedactionSecretsFor($service), + ); return response()->json([ 'logs' => $logs, @@ -1132,6 +1144,7 @@ class ServicesController extends Controller ], 'force_domain_override' => ['type' => 'boolean', 'default' => false, 'description' => 'Force domain override even if conflicts are detected.'], 'is_container_label_escape_enabled' => ['type' => 'boolean', 'default' => true, 'description' => 'Escape special characters in labels. By default, $ (and other chars) is escaped. If you want to use env variables inside the labels, turn this off.'], + 'redact_all_env_values_in_logs' => ['type' => 'boolean', 'default' => true, 'description' => 'Redact all environment variable values in logs. When false, only locked (shown once) values are redacted. Requires team admin permissions.'], ], ) ), @@ -1220,12 +1233,16 @@ class ServicesController extends Controller $this->authorize('update', $service); + if ($request->has('redact_all_env_values_in_logs')) { + $this->authorize('manageEnvironment', $service); + } + if ($request->boolean('instant_deploy')) { abort_unless($request->user()->tokenCan('deploy') || $request->user()->tokenCan('root'), 403, 'Missing required permissions: deploy'); $this->authorize('deploy', $service); } - $allowedFields = ['name', 'description', 'instant_deploy', 'docker_compose_raw', 'connect_to_docker_network', 'urls', 'force_domain_override', 'is_container_label_escape_enabled']; + $allowedFields = ['name', 'description', 'instant_deploy', 'docker_compose_raw', 'connect_to_docker_network', 'urls', 'force_domain_override', 'is_container_label_escape_enabled', 'redact_all_env_values_in_logs']; $validationRules = [ 'name' => 'string|max:255', @@ -1239,6 +1256,7 @@ class ServicesController extends Controller 'urls.*.url' => ValidationPatterns::applicationDomainRules(), 'force_domain_override' => 'boolean', 'is_container_label_escape_enabled' => 'boolean', + 'redact_all_env_values_in_logs' => 'boolean', ]; $validationMessages = [ 'urls.*.array' => 'An item in the urls array has invalid fields. Only name and url fields are supported.', @@ -1306,6 +1324,9 @@ class ServicesController extends Controller if ($request->has('is_container_label_escape_enabled')) { $service->is_container_label_escape_enabled = $request->boolean('is_container_label_escape_enabled'); } + if ($request->has('redact_all_env_values_in_logs')) { + $service->redact_all_env_values_in_logs = $request->boolean('redact_all_env_values_in_logs'); + } $previousDnsHostnames = $this->serviceApplicationDnsHostnames($service); $service->save(); diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index 8e3d0437ee..c429a8ae20 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -6025,7 +6025,9 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); { $this->failDeployment(); - // Log comprehensive error information + // failed() runs on a fresh job instance, so remote secrets fetched during the deployment are not loaded here. + // Every entry still passes through ApplicationDeploymentQueue::redactLogText(); raw stack traces are not written + // because their truncated arguments can hold partial secrets that redaction cannot match. $errorMessage = $exception->getMessage() ?: 'Unknown error occurred'; $errorCode = $exception->getCode(); $errorClass = get_class($exception); @@ -6053,14 +6055,6 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); $this->application_deployment_queue->addLogEntry(" {$previousClass}: {$previousMessage}", 'stderr', hidden: true); $this->application_deployment_queue->addLogEntry(" at {$previous->getFile()}:{$previous->getLine()}", 'stderr', hidden: true); } - - // Log first few lines of stack trace for debugging - $trace = $exception->getTraceAsString(); - $traceLines = explode("\n", $trace); - $this->application_deployment_queue->addLogEntry('Stack trace (first 5 lines):', 'stderr', hidden: true); - foreach (array_slice($traceLines, 0, 5) as $traceLine) { - $this->application_deployment_queue->addLogEntry(" {$traceLine}", 'stderr', hidden: true); - } $this->application_deployment_queue->addLogEntry('========================================', 'stderr'); if ($this->application->build_pack !== 'dockercompose') { diff --git a/app/Jobs/CoolifyTask.php b/app/Jobs/CoolifyTask.php index cac687fc85..fe7237ccf4 100755 --- a/app/Jobs/CoolifyTask.php +++ b/app/Jobs/CoolifyTask.php @@ -99,14 +99,17 @@ class CoolifyTask implements ShouldBeEncrypted, ShouldQueue 'job' => 'CoolifyTask', 'activity_id' => $this->activity->id, 'server_uuid' => $this->activity->getExtraProperty('server_uuid'), - 'error' => $exception?->getMessage(), + 'error' => $exception ? remove_iip($exception->getMessage()) : null, 'total_attempts' => $this->attempts(), 'trace' => $exception?->getTraceAsString(), ]); // A database import whose restore can still run in the database container stays in progress // until its stop has run, so it keeps blocking other operations on the database. - if (DatabaseImportCleanup::stopAfterTaskFailure($this->activity, $exception?->getMessage() ?: 'Job permanently failed')) { + // The message can contain command output or the SSH command, so store only a redacted copy. + $message = filled($exception?->getMessage()) ? remove_iip($exception->getMessage()) : null; + + if (DatabaseImportCleanup::stopAfterTaskFailure($this->activity, $message ?: 'Job permanently failed')) { RemoteProcessCommand::forget($this->activity); return; @@ -118,7 +121,7 @@ class CoolifyTask implements ShouldBeEncrypted, ShouldQueue $stopRequested = DatabaseImportCleanup::stopRequested($this->activity); $this->activity->properties = $this->activity->properties->merge(array_filter([ 'status' => ProcessStatus::ERROR->value, - 'error' => $stopRequested ? null : ($exception?->getMessage() ?? 'Job permanently failed'), + 'error' => $stopRequested ? null : ($message ?? 'Job permanently failed'), 'failed_at' => now()->toIso8601String(), ], fn ($value) => $value !== null)); $this->activity->save(); diff --git a/app/Jobs/DatabaseBackupJob.php b/app/Jobs/DatabaseBackupJob.php index 00f3d35a50..ebdc947e97 100644 --- a/app/Jobs/DatabaseBackupJob.php +++ b/app/Jobs/DatabaseBackupJob.php @@ -20,6 +20,7 @@ use App\Notifications\Database\BackupSuccess; use App\Notifications\Database\BackupSuccessWithS3Warning; use App\Rules\SafeWebhookUrl; use App\Services\ScheduledJobDeliveryService; +use App\Services\Security\SensitiveDataRedactor; use App\Support\BackupCompression; use App\Support\ClickhouseBackupCommand; use App\Support\DatabaseOperationReservation; @@ -454,10 +455,11 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue } catch (Throwable $e) { // Local backup failed: the partial file is not tracked, so retention would never remove it. deleteBackupsLocally($this->backup_location, $this->server); + $failureMessage = $this->redact($this->error_output ?? $this->backup_output ?? $e->getMessage()); if ($this->backup_log) { $this->backup_log->update([ 'status' => 'failed', - 'message' => $this->error_output ?? $this->backup_output ?? $e->getMessage(), + 'message' => $failureMessage, 'size' => $size, 'filename' => null, 's3_uploaded' => null, @@ -465,7 +467,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue ]); } try { - $this->team?->notify(new BackupFailed($this->backup, $this->database, $this->error_output ?? $this->backup_output ?? $e->getMessage(), $database)); + $this->team?->notify(new BackupFailed($this->backup, $this->database, $failureMessage, $database)); } catch (Throwable $notifyException) { Log::channel('scheduled-errors')->warning('Failed to send backup failure notification', [ 'backup_id' => $this->backup->uuid, @@ -490,13 +492,13 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue } } catch (Throwable $e) { // S3 upload failed but local backup succeeded - $s3UploadError = $e->getMessage(); + $s3UploadError = $this->redact($e->getMessage()); } } // Step 3: Update status and send notifications based on results if ($localBackupSucceeded) { - $message = $this->backup_output; + $message = $this->backup_output === null ? null : $this->redact($this->backup_output); if ($s3UploadError) { $message = $message @@ -867,7 +869,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue $this->upload_to_s3($storage); $uploaded = true; } catch (Throwable $e) { - $uploadError = $e->getMessage(); + $uploadError = $this->redact($e->getMessage()); $failures[] = "{$storage->name}: {$uploadError}"; } @@ -878,7 +880,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue 'message' => $uploadError, ]); } catch (Throwable $e) { - $failures[] = "{$storage->name}: {$e->getMessage()}"; + $failures[] = "{$storage->name}: {$this->redact($e->getMessage())}"; } } @@ -1100,7 +1102,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue foreach ($unfinished as $execution) { $execution->update([ 'status' => 'failed', - 'message' => 'Job permanently failed after '.$this->attempts().' attempts: '.($exception?->getMessage() ?? 'Unknown error'), + 'message' => $this->redact('Job permanently failed after '.$this->attempts().' attempts: '.($exception?->getMessage() ?? 'Unknown error')), 'size' => 0, 'filename' => null, 'finished_at' => Carbon::now(), @@ -1111,7 +1113,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue return; } - $output = $this->backup_output ?? $exception?->getMessage() ?? 'Unknown error'; + $output = $this->redact($this->backup_output ?? $exception?->getMessage() ?? 'Unknown error'); $databaseNames = $unfinished->isEmpty() ? ['unknown'] : $unfinished->map(fn (ScheduledDatabaseBackupExecution $execution) => $execution->database_name ?? 'unknown'); foreach ($databaseNames as $databaseName) { try { @@ -1125,6 +1127,31 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue } } + /** + * Redact the backed-up database credentials and the key and secret of every S3 destination from text that is stored or sent in notifications. + */ + private function redact(string $text): string + { + try { + $database = $this->database ?? $this->backup->database; + $s3 = $this->s3 ?? $this->backup->s3; + $lockedSecrets = [$this->postgres_password, $this->mongo_root_password, $s3?->key, $s3?->secret]; + foreach ($this->backup->selectedS3Storages() as $storage) { + $lockedSecrets[] = $storage->key; + $lockedSecrets[] = $storage->secret; + } + foreach (array_keys($database?->getAttributes() ?? []) as $attribute) { + if (str_contains($attribute, 'password')) { + $lockedSecrets[] = $database->getAttribute($attribute); + } + } + } catch (Throwable) { + return SensitiveDataRedactor::FAILURE_MESSAGE; + } + + return app(SensitiveDataRedactor::class)->redactText($text, lockedSecrets: array_filter($lockedSecrets, fn ($secret): bool => is_string($secret) && $secret !== '')); + } + /** * @return Collection */ diff --git a/app/Jobs/DeleteResourceJob.php b/app/Jobs/DeleteResourceJob.php index b73c2c5a7a..d5e8aaf4ff 100644 --- a/app/Jobs/DeleteResourceJob.php +++ b/app/Jobs/DeleteResourceJob.php @@ -115,7 +115,7 @@ class DeleteResourceJob implements ShouldBeEncrypted, ShouldQueue $server = data_get($this->resource, 'destination.server'); if ($server) { $this->resource->team()?->notify(new GeneralNotification( - "'{$this->resource->name}' was removed from Coolify, but its Docker resources could not be removed from server '{$server->name}' and may still exist there. Error: {$e->getMessage()}", + "'{$this->resource->name}' was removed from Coolify, but its Docker resources could not be removed from server '{$server->name}' and may still exist there. Error: ".remove_iip($e->getMessage()), success: false, )); } diff --git a/app/Jobs/DockerCleanupJob.php b/app/Jobs/DockerCleanupJob.php index 1a338f093f..ed430a600c 100644 --- a/app/Jobs/DockerCleanupJob.php +++ b/app/Jobs/DockerCleanupJob.php @@ -187,14 +187,15 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue } } catch (\Throwable $e) { $failed = true; + $errorMessage = remove_iip($e->getMessage()); if ($this->execution_log) { $this->execution_log->update([ 'status' => 'failed', - 'message' => $e->getMessage(), + 'message' => $errorMessage, ]); event(new DockerCleanupDone($this->execution_log)); } - $this->server->team?->notify(new DockerCleanupFailed($this->server, 'Docker cleanup job failed with the following error: '.$e->getMessage())); + $this->server->team?->notify(new DockerCleanupFailed($this->server, 'Docker cleanup job failed with the following error: '.$errorMessage)); throw $e; } finally { if (! $failed && $this->occurrenceUuid) { @@ -246,7 +247,7 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue return; } - $message = $exception?->getMessage() ?? 'Docker cleanup job failed without an exception.'; + $message = remove_iip($exception?->getMessage() ?? 'Docker cleanup job failed without an exception.'); $updated = DockerCleanupExecution::query() ->whereKey($execution->id) diff --git a/app/Jobs/ProvisionGithubRunnerJob.php b/app/Jobs/ProvisionGithubRunnerJob.php index c5b1148448..e4415a81c1 100644 --- a/app/Jobs/ProvisionGithubRunnerJob.php +++ b/app/Jobs/ProvisionGithubRunnerJob.php @@ -120,7 +120,7 @@ class ProvisionGithubRunnerJob implements ShouldBeEncrypted, ShouldQueue ->where('status', GithubRunnerStatus::Provisioning) ->update([ 'status' => GithubRunnerStatus::Failed, - 'error_message' => "Could not start a runner after {$attempt} attempts: {$e->getMessage()}", + 'error_message' => remove_iip("Could not start a runner after {$attempt} attempts: {$e->getMessage()}"), 'completed_at' => now(), ]); CleanupGithubRunnerJob::dispatch($execution->id); @@ -140,7 +140,7 @@ class ProvisionGithubRunnerJob implements ShouldBeEncrypted, ShouldQueue 'runner_name' => null, 'runner_id' => null, 'ready_at' => null, - 'error_message' => "Attempt {$attempt} of ".self::MAX_ATTEMPTS." failed: {$e->getMessage()}", + 'error_message' => remove_iip("Attempt {$attempt} of ".self::MAX_ATTEMPTS." failed: {$e->getMessage()}"), ]); if (! $retried) { return; diff --git a/app/Jobs/ScheduledTaskJob.php b/app/Jobs/ScheduledTaskJob.php index 8538760db8..56c5bb7821 100644 --- a/app/Jobs/ScheduledTaskJob.php +++ b/app/Jobs/ScheduledTaskJob.php @@ -5,6 +5,7 @@ namespace App\Jobs; use App\Events\ScheduledTaskDone; use App\Exceptions\NonReportableException; use App\Models\Application; +use App\Models\EnvironmentVariable; use App\Models\ScheduledTask; use App\Models\ScheduledTaskExecution; use App\Models\Server; @@ -13,6 +14,7 @@ use App\Models\Team; use App\Notifications\ScheduledTask\TaskFailed; use App\Notifications\ScheduledTask\TaskSuccess; use App\Services\ScheduledJobDeliveryService; +use App\Services\Security\SensitiveDataRedactor; use Carbon\Carbon; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeEncrypted; @@ -67,6 +69,9 @@ class ScheduledTaskJob implements ShouldBeEncrypted, ShouldQueue public ?string $task_output = null; + /** @var array{known: array, locked: array}|null */ + private ?array $logRedactionSecrets = null; + public array $containers = []; public string $server_timezone = 'UTC'; @@ -178,7 +183,7 @@ class ScheduledTaskJob implements ShouldBeEncrypted, ShouldQueue $exec = $this->boundedTaskCommand($execCommand); // Disable SSH multiplexing to prevent race conditions when multiple tasks run concurrently // See: https://github.com/coollabsio/coolify/issues/6736 - $this->task_output = instant_remote_process([$exec], $this->server, throwError: true, no_sudo: true, timeout: $this->task->timeout ?? 300, disableMultiplexing: true); + $this->task_output = $this->redact(instant_remote_process([$exec], $this->server, throwError: true, no_sudo: true, timeout: $this->task->timeout ?? 300, disableMultiplexing: true) ?? ''); $this->task_log->update([ 'status' => 'success', 'message' => $this->task_output, @@ -194,10 +199,11 @@ class ScheduledTaskJob implements ShouldBeEncrypted, ShouldQueue throw new NonReportableException('ScheduledTaskJob failed: No valid container was found. Is the container name correct?'); } catch (\Throwable $e) { $failed = true; + $safeErrorMessage = $this->redact($e->getMessage()); if ($this->task_log) { $this->task_log->update([ 'status' => 'failed', - 'message' => $this->task_output ?? $e->getMessage(), + 'message' => $this->task_output ?? $safeErrorMessage, ]); } @@ -208,13 +214,13 @@ class ScheduledTaskJob implements ShouldBeEncrypted, ShouldQueue 'task_name' => $this->task->name, 'server' => $this->server?->name ?? 'unknown', 'attempt' => $this->attempts(), - 'error' => $e->getMessage(), + 'error' => $safeErrorMessage, ]); // Only notify and throw on final failure // Re-throw to trigger Laravel's retry mechanism with backoff - throw $e; + throw $this->redactedException($e, $safeErrorMessage); } finally { if (! $failed && $this->occurrenceUuid) { app(ScheduledJobDeliveryService::class)->complete($this->occurrenceUuid, $this->job?->uuid() ?? $this->occurrenceUuid); @@ -244,6 +250,32 @@ class ScheduledTaskJob implements ShouldBeEncrypted, ShouldQueue return "output_file=\$(mktemp); trap 'rm -f \"\$output_file\"' EXIT; set +e; set -o pipefail; {$command} 2>&1 | { head -c {$readLimit} > \"\$output_file\"; cat > /dev/null; }; exit_code=\${PIPESTATUS[0]}; if [ \"\$(wc -c < \"\$output_file\")\" -gt {$maxOutputBytes} ]; then truncate -s {$maxOutputBytes} \"\$output_file\"; printf '\n\n[... Output truncated at 5MB limit ...]' >> \"\$output_file\"; fi; if [ \"\$exit_code\" -eq 0 ]; then cat \"\$output_file\"; else cat \"\$output_file\" >&2; fi; exit \$exit_code"; } + private function redact(string $value): string + { + try { + $this->logRedactionSecrets ??= EnvironmentVariable::logRedactionSecretsFor($this->resource); + } catch (\Throwable) { + return REDACTED; + } + + return resolve(SensitiveDataRedactor::class)->redactText($value, $this->logRedactionSecrets['known'], $this->logRedactionSecrets['locked']); + } + + /** + * Keep the original exception when redaction did not change its message. + * Otherwise, drop the original (and its trace) so the secret is not stored in failed_jobs. + */ + private function redactedException(\Throwable $exception, string $safeMessage): \Throwable + { + if ($safeMessage === $exception->getMessage()) { + return $exception; + } + + return $exception instanceof NonReportableException + ? new NonReportableException($safeMessage, (int) $exception->getCode()) + : new \RuntimeException($safeMessage, (int) $exception->getCode()); + } + /** * Calculate the number of seconds to wait before retrying the job. */ @@ -262,6 +294,9 @@ class ScheduledTaskJob implements ShouldBeEncrypted, ShouldQueue } $this->team ??= Team::find($this->task->team_id); + $this->resource ??= $this->task->application ?? $this->task->service; + $safeErrorMessage = $this->redact($exception?->getMessage() ?? 'Unknown error'); + $safeTrace = $this->redact($exception?->getTraceAsString() ?? ''); Log::channel('scheduled-errors')->error('ScheduledTask permanently failed', [ 'job' => 'ScheduledTaskJob', @@ -269,8 +304,8 @@ class ScheduledTaskJob implements ShouldBeEncrypted, ShouldQueue 'task_name' => $this->task->name, 'server' => $this->server?->name ?? 'unknown', 'total_attempts' => $this->attempts(), - 'error' => $exception?->getMessage(), - 'trace' => $exception?->getTraceAsString(), + 'error' => $safeErrorMessage, + 'trace' => $safeTrace, ]); // After a worker timeout only the queued uuid identifies this run's execution. @@ -282,13 +317,13 @@ class ScheduledTaskJob implements ShouldBeEncrypted, ShouldQueue if ($execution) { $errorMessage = 'Job permanently failed after '.$this->attempts().' attempts'; if ($exception) { - $errorMessage .= ': '.$exception->getMessage(); + $errorMessage .= ': '.$safeErrorMessage; } $execution->update([ 'status' => 'failed', 'message' => $errorMessage, - 'error_details' => $exception?->getTraceAsString(), + 'error_details' => $safeTrace, 'finished_at' => Carbon::now()->toImmutable(), ]); } else { @@ -299,6 +334,6 @@ class ScheduledTaskJob implements ShouldBeEncrypted, ShouldQueue } // Notify team about permanent failure - $this->team?->notify(new TaskFailed($this->task, $exception?->getMessage() ?? 'Unknown error')); + $this->team?->notify(new TaskFailed($this->task, $safeErrorMessage)); } } diff --git a/app/Jobs/SendMessageToDiscordJob.php b/app/Jobs/SendMessageToDiscordJob.php index 226e9f4cd7..eeae3bf64a 100644 --- a/app/Jobs/SendMessageToDiscordJob.php +++ b/app/Jobs/SendMessageToDiscordJob.php @@ -72,7 +72,7 @@ class SendMessageToDiscordJob implements ShouldBeEncrypted, ShouldQueue return; } - $response = Http::withOptions($httpOptions)->post($this->webhookUrl, $this->message->toPayload()); + $response = $this->sendNotificationRequest('Discord', $this->webhookUrl, fn () => Http::withOptions($httpOptions)->post($this->webhookUrl, $this->message->toPayload())); // 401: invalid webhook token, 403: missing access, 404: unknown (deleted) webhook. $this->handleNotificationResponse($response, 'Discord', in_array($response->status(), [401, 403, 404], true)); diff --git a/app/Jobs/SendMessageToPushoverJob.php b/app/Jobs/SendMessageToPushoverJob.php index 406a7e5ea4..f7ff676d56 100644 --- a/app/Jobs/SendMessageToPushoverJob.php +++ b/app/Jobs/SendMessageToPushoverJob.php @@ -45,7 +45,7 @@ class SendMessageToPushoverJob implements ShouldBeEncrypted, ShouldQueue */ public function handle(): void { - $response = Http::post('https://api.pushover.net/1/messages.json', $this->message->toPayload($this->token, $this->user)); + $response = $this->sendNotificationRequest('Pushover', 'https://api.pushover.net/1/messages.json', fn () => Http::post('https://api.pushover.net/1/messages.json', $this->message->toPayload($this->token, $this->user))); // Pushover marks the invalid field in the error body, for example {"token": "invalid"}. $isConfigurationError = $response->clientError() diff --git a/app/Jobs/SendMessageToSlackJob.php b/app/Jobs/SendMessageToSlackJob.php index 0b01f7f658..0daafcc42e 100644 --- a/app/Jobs/SendMessageToSlackJob.php +++ b/app/Jobs/SendMessageToSlackJob.php @@ -71,9 +71,9 @@ class SendMessageToSlackJob implements ShouldBeEncrypted, ShouldQueue * * @see https://github.com/coollabsio/coolify/pull/6139#issuecomment-3756777708 */ - $response = $this->isSlackWebhook() + $response = $this->sendNotificationRequest('Slack', $this->webhookUrl, fn () => $this->isSlackWebhook() ? $this->sendToSlack($httpOptions) - : $this->sendToMattermost($httpOptions); + : $this->sendToMattermost($httpOptions)); // 403: posting is not allowed, 404: the webhook or channel was removed, 410: the channel is archived. $this->handleNotificationResponse($response, 'Slack', in_array($response->status(), [403, 404, 410], true)); diff --git a/app/Jobs/SendMessageToTelegramJob.php b/app/Jobs/SendMessageToTelegramJob.php index 0cb1a7124d..c6bf073b56 100644 --- a/app/Jobs/SendMessageToTelegramJob.php +++ b/app/Jobs/SendMessageToTelegramJob.php @@ -78,7 +78,7 @@ class SendMessageToTelegramJob implements ShouldBeEncrypted, ShouldQueue if ($this->threadId) { $payload['message_thread_id'] = $this->threadId; } - $response = Http::post($url, $payload); + $response = $this->sendNotificationRequest('Telegram', $url, fn () => Http::post($url, $payload)); // 401: invalid bot token, 403: the bot was blocked or removed from the chat, 400 "chat not found": wrong chat ID. $isConfigurationError = in_array($response->status(), [401, 403], true) diff --git a/app/Jobs/SendWebhookJob.php b/app/Jobs/SendWebhookJob.php index 29ee58fb2f..11d44eceb2 100644 --- a/app/Jobs/SendWebhookJob.php +++ b/app/Jobs/SendWebhookJob.php @@ -71,7 +71,7 @@ class SendWebhookJob implements ShouldBeEncrypted, ShouldQueue return; } - $response = Http::withOptions($httpOptions)->post($this->webhookUrl, $this->payload); + $response = $this->sendNotificationRequest('Webhook', $this->webhookUrl, fn () => Http::withOptions($httpOptions)->post($this->webhookUrl, $this->payload)); // 401/403: the endpoint rejects us, 404/410: the endpoint does not exist anymore. $this->handleNotificationResponse($response, 'Webhook', in_array($response->status(), [401, 403, 404, 410], true)); diff --git a/app/Jobs/VolumeBackupJob.php b/app/Jobs/VolumeBackupJob.php index fe27be0441..9c36f67e53 100644 --- a/app/Jobs/VolumeBackupJob.php +++ b/app/Jobs/VolumeBackupJob.php @@ -13,6 +13,7 @@ use App\Notifications\VolumeBackup\BackupFailed; use App\Notifications\VolumeBackup\BackupSuccess; use App\Rules\SafeWebhookUrl; use App\Services\ScheduledJobDeliveryService; +use App\Services\Security\SensitiveDataRedactor; use App\Support\BackupCompression; use Carbon\Carbon; use Illuminate\Bus\Queueable; @@ -188,6 +189,7 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue } } + $warning = $warning === null ? null : $this->redact($warning); $this->execution->update([ 'status' => 'success', 'message' => $warning, @@ -203,7 +205,7 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue Log::channel('scheduled-errors')->warning('Volume backup retention cleanup failed', [ 'backup_id' => $this->backup->id, 'execution_id' => $this->execution->id, - 'error' => $exception->getMessage(), + 'error' => $this->redact($exception->getMessage()), ]); } } catch (Throwable $exception) { @@ -234,14 +236,17 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue $this->execution->update([ 'status' => 'failed', - 'message' => $exception->getMessage().$recoveryError, + 'message' => $this->redact($exception->getMessage().$recoveryError), 'filename' => $streamToS3 ? ($s3CleanupPending ? $backupLocation : null) : ($archiveDeleted && ! $s3CleanupPending ? null : $backupLocation), 'local_storage_deleted' => $archiveDeleted, ]); - throw $exception; + $safeMessage = $this->redact($exception->getMessage()); + throw $safeMessage === $exception->getMessage() + ? $exception + : new \RuntimeException($safeMessage, (int) $exception->getCode()); } finally { if (! $failed && $this->occurrenceUuid) { app(ScheduledJobDeliveryService::class)->complete($this->occurrenceUuid, $this->job?->uuid() ?? $this->occurrenceUuid); @@ -285,7 +290,7 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue $execution->update([ 'status' => 'failed', - 'message' => ($exception?->getMessage() ?? 'Volume backup timed out or was terminated.').$recoveryError, + 'message' => $this->redact(($exception?->getMessage() ?? 'Volume backup timed out or was terminated.').$recoveryError), 'finished_at' => now(), 'filename' => $filename, 'local_storage_deleted' => $localStorageDeleted, @@ -294,10 +299,29 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue $this->backup->team?->notify(new BackupFailed( $this->backup, - $exception?->getMessage() ?? 'Volume backup timed out or was terminated.', + $this->redact($exception?->getMessage() ?? 'Volume backup timed out or was terminated.'), )); } + /** + * Redact the key and secret of every S3 destination of the backup from text that is stored or sent in notifications. + */ + private function redact(string $text): string + { + try { + $s3 = $this->backup->s3; + $lockedSecrets = [$s3?->key, $s3?->secret]; + foreach ($this->backup->selectedS3Storages() as $storage) { + $lockedSecrets[] = $storage->key; + $lockedSecrets[] = $storage->secret; + } + } catch (Throwable) { + return SensitiveDataRedactor::FAILURE_MESSAGE; + } + + return app(SensitiveDataRedactor::class)->redactText($text, lockedSecrets: array_filter($lockedSecrets, fn ($secret): bool => is_string($secret) && $secret !== '')); + } + /** * @return array */ @@ -376,8 +400,9 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue $this->uploadToS3($replica->s3, $backupLocation, $backupDirectory, $server); $replica->update(['s3_uploaded' => true, 'message' => null]); } catch (Throwable $exception) { - $replica->update(['s3_uploaded' => false, 'message' => $exception->getMessage()]); - $errors[] = ($replica->s3?->name ?? 'Deleted storage').': '.$exception->getMessage(); + $message = $this->redact($exception->getMessage()); + $replica->update(['s3_uploaded' => false, 'message' => $message]); + $errors[] = ($replica->s3?->name ?? 'Deleted storage').': '.$message; } } diff --git a/app/Livewire/Project/Shared/EnvironmentVariable/All.php b/app/Livewire/Project/Shared/EnvironmentVariable/All.php index 4a2b894c60..daad018b2b 100644 --- a/app/Livewire/Project/Shared/EnvironmentVariable/All.php +++ b/app/Livewire/Project/Shared/EnvironmentVariable/All.php @@ -57,6 +57,8 @@ class All extends Component public bool $use_build_secrets = false; + public bool $redact_all_env_values_in_logs = true; + /** * Environment variable rows are loaded after first paint via wire:init * so the surrounding configuration page can render immediately. @@ -109,6 +111,9 @@ class All extends Component { $this->is_env_sorting_enabled = data_get($this->resource, 'settings.is_env_sorting_enabled', false); $this->use_build_secrets = data_get($this->resource, 'settings.use_build_secrets', false); + if ($this->hasLogRedactionSetting()) { + $this->redact_all_env_values_in_logs = $this->resource->redactsAllEnvValuesInLogs(); + } $this->resourceClass = get_class($this->resource); $resourceWithPreviews = [Application::class]; $simpleDockerfile = filled(data_get($this->resource, 'dockerfile')); @@ -137,6 +142,41 @@ class All extends Component } } + /** + * Applications, services, and standalone databases have the "Environment values in logs" setting. + */ + public function hasLogRedactionSetting(): bool + { + return method_exists($this->resource, 'redactsAllEnvValuesInLogs'); + } + + public function saveLogRedactionSetting() + { + try { + $this->authorize('manageEnvironment', $this->resource); + + if ($this->resource instanceof Application) { + $this->resource->settings->redact_all_env_values_in_logs = $this->redact_all_env_values_in_logs; + $this->saveApplicationSettingsWithAudit($this->resource); + } elseif ($this->hasLogRedactionSetting()) { + // Services and databases use the Auditable trait, so saving records the changed field. + $this->resource->redact_all_env_values_in_logs = $this->redact_all_env_values_in_logs; + $this->resource->save(); + } else { + return; + } + + $this->dispatch('success', 'Log redaction setting updated.'); + } catch (\Throwable $e) { + if ($this->hasLogRedactionSetting()) { + $this->resource->refresh(); + $this->redact_all_env_values_in_logs = $this->resource->redactsAllEnvValuesInLogs(); + } + + return handleError($e, $this); + } + } + public function instantSave() { try { diff --git a/app/Livewire/Project/Shared/GetLogs.php b/app/Livewire/Project/Shared/GetLogs.php index e809c1aae5..f9b15afac4 100644 --- a/app/Livewire/Project/Shared/GetLogs.php +++ b/app/Livewire/Project/Shared/GetLogs.php @@ -4,6 +4,7 @@ namespace App\Livewire\Project\Shared; use App\Helpers\SshMultiplexingHelper; use App\Models\Application; +use App\Models\EnvironmentVariable; use App\Models\Server; use App\Models\Service; use App\Models\ServiceApplication; @@ -192,7 +193,7 @@ class GetLogs extends Component $options = "-n {$this->resolveLogTail()}".($this->showTimeStamps ? ' -t' : ''); - return sanitizeLogsForExport($this->runLogCommand($this->logCommand($options), self::MAX_DISPLAY_SIZE_BYTES, '5MB', $this->showTimeStamps)); + return sanitizeLogsForExport($this->runLogCommand($this->logCommand($options), self::MAX_DISPLAY_SIZE_BYTES, '5MB', $this->showTimeStamps), $this->logRedactionSecrets()); } public function downloadAllLogs(): string @@ -212,7 +213,19 @@ class GetLogs extends Component $command = $this->logCommand($this->showTimeStamps ? '-t' : ''); - return sanitizeLogsForExport($this->runLogCommand($command, self::MAX_DOWNLOAD_SIZE_BYTES, '50MB', $this->showTimeStamps)); + return sanitizeLogsForExport($this->runLogCommand($command, self::MAX_DOWNLOAD_SIZE_BYTES, '50MB', $this->showTimeStamps), $this->logRedactionSecrets()); + } + + /** + * Environment values and database credentials of the resource whose logs are exported. + * + * @return array{known: array, locked: array} + */ + private function logRedactionSecrets(): array + { + $resource = $this->resource ?? $this->servicesubtype?->service; + + return EnvironmentVariable::logRedactionSecretsFor($resource, includePreview: $resource instanceof Application && filled($this->pull_request)); } private function resolveLogTail(): int|string diff --git a/app/Livewire/Server/DockerCleanupExecutions.php b/app/Livewire/Server/DockerCleanupExecutions.php index 6cb76fc9f3..763115aa20 100644 --- a/app/Livewire/Server/DockerCleanupExecutions.php +++ b/app/Livewire/Server/DockerCleanupExecutions.php @@ -4,12 +4,13 @@ namespace App\Livewire\Server; use App\Models\Server; use App\Traits\ListensToTeamChannel; +use Illuminate\Foundation\Auth\Access\AuthorizesRequests; use Illuminate\Support\Collection; use Livewire\Component; class DockerCleanupExecutions extends Component { - use ListensToTeamChannel; + use AuthorizesRequests, ListensToTeamChannel; public Server $server; @@ -34,12 +35,15 @@ class DockerCleanupExecutions extends Component public function mount(Server $server) { + $this->authorize('view', $server); $this->server = $server; $this->refreshExecutions(); } public function refreshExecutions(): void { + $this->authorize('view', $this->server); + $this->executions = $this->server->dockerCleanupExecutions() ->orderBy('created_at', 'desc') ->take(20) @@ -55,6 +59,8 @@ class DockerCleanupExecutions extends Component public function selectExecution($key): void { + $this->authorize('view', $this->server); + if ($key == $this->selectedKey) { $this->selectedKey = null; $this->selectedExecution = null; @@ -74,6 +80,8 @@ class DockerCleanupExecutions extends Component public function polling() { + $this->authorize('view', $this->server); + if ($this->selectedExecution && $this->isPollingActive) { $this->selectedExecution->refresh(); if ($this->selectedExecution->status !== 'running') { @@ -85,6 +93,8 @@ class DockerCleanupExecutions extends Component public function loadMoreLogs() { + $this->authorize('view', $this->server); + $this->currentPage++; } @@ -105,6 +115,8 @@ class DockerCleanupExecutions extends Component public function downloadLogs(int $executionId) { + $this->authorize('view', $this->server); + $execution = $this->executions->firstWhere('id', $executionId); if (! $execution) { return; diff --git a/app/Logging/RedactSensitiveAuditData.php b/app/Logging/RedactSensitiveAuditData.php new file mode 100644 index 0000000000..901325c0cf --- /dev/null +++ b/app/Logging/RedactSensitiveAuditData.php @@ -0,0 +1,19 @@ +redactor->withoutEmailRedaction()))($logger); + } +} diff --git a/app/Logging/RedactSensitiveData.php b/app/Logging/RedactSensitiveData.php new file mode 100644 index 0000000000..4b58280f33 --- /dev/null +++ b/app/Logging/RedactSensitiveData.php @@ -0,0 +1,76 @@ +getLogger()->pushProcessor(fn (LogRecord $record): LogRecord => $this->redact($record)); + } + + /** + * Self-hosted instances may turn off framework log redaction. Cloud always redacts. + */ + public static function enabled(): bool + { + if (! app()->bound('config')) { + return true; + } + + return isCloud() || config('logging.redaction_enabled', true) !== false; + } + + private function redact(LogRecord $record): LogRecord + { + try { + $message = $this->redactor->redactText($record->message); + $recordContext = $this->redactor->redactValue($this->normalizeThrowables($record->context)); + $extra = $this->redactor->redactValue($this->normalizeThrowables($record->extra)); + + return $record->with( + message: $message, + context: is_array($recordContext) ? $recordContext : [], + extra: is_array($extra) ? $extra : [], + ); + } catch (Throwable) { + return $record->with( + message: SensitiveDataRedactor::FAILURE_MESSAGE, + context: [], + extra: [], + ); + } + } + + private function normalizeThrowables(mixed $value): mixed + { + if ($value instanceof Throwable) { + return [ + 'class' => $value::class, + 'message' => $value->getMessage(), + 'code' => $value->getCode(), + 'file' => $value->getFile(), + 'line' => $value->getLine(), + 'trace' => $value->getTraceAsString(), + 'previous' => $this->normalizeThrowables($value->getPrevious()), + ]; + } + + if (! is_array($value)) { + return $value; + } + + return array_map($this->normalizeThrowables(...), $value); + } +} diff --git a/app/Mcp/Concerns/BuildsResponse.php b/app/Mcp/Concerns/BuildsResponse.php index f6311f22bb..e2d3e3dfea 100644 --- a/app/Mcp/Concerns/BuildsResponse.php +++ b/app/Mcp/Concerns/BuildsResponse.php @@ -2,6 +2,7 @@ namespace App\Mcp\Concerns; +use App\Services\Security\SensitiveDataRedactor; use Laravel\Mcp\Request; use Laravel\Mcp\Response; @@ -108,27 +109,13 @@ trait BuildsResponse /** * Best-effort redaction for free-form log text shown to MCP clients. - * Uses remove_iip plus common KEY=value / JSON secret patterns; not a guarantee. + * Uses SensitiveDataRedactor patterns plus the given resource values; not a guarantee. + * + * @param array{known?: array, locked?: array} $secrets Values from EnvironmentVariable::logRedactionSecretsFor(). */ - protected function redactLogText(string $text): string + protected function redactLogText(string $text, array $secrets = []): string { - $text = remove_iip($text); - - // password= / secret= / token= / "token":"..." style (shell or JSON; quoted or bare) - $text = preg_replace( - '/(?redactText($text, $secrets['known'] ?? [], $secrets['locked'] ?? []); } /** diff --git a/app/Mcp/Tools/Control.php b/app/Mcp/Tools/Control.php index 65ec7ca4c1..debbb4a080 100644 --- a/app/Mcp/Tools/Control.php +++ b/app/Mcp/Tools/Control.php @@ -13,6 +13,7 @@ use App\Mcp\Concerns\BuildsResponse; use App\Mcp\Concerns\ResolvesResource; use App\Mcp\Concerns\ResolvesTeam; use App\Models\Application; +use App\Models\EnvironmentVariable; use App\Support\ResourceStartActivity; use Illuminate\Contracts\JsonSchema\JsonSchema; use Laravel\Mcp\Request; @@ -70,7 +71,9 @@ class Control extends Tool 'service' => $this->controlService($resource, $action), }; } catch (\Throwable $e) { - return $this->mcpError($request, $e->getMessage(), ['resource_uuid' => $uuid]); + $secrets = EnvironmentVariable::logRedactionSecretsFor($resource); + + return $this->mcpError($request, $this->redactLogText($e->getMessage(), $secrets), ['resource_uuid' => $uuid]); } auditLog('mcp.control', [ diff --git a/app/Mcp/Tools/GetLogs.php b/app/Mcp/Tools/GetLogs.php index b90df64caf..4d65c0c38f 100644 --- a/app/Mcp/Tools/GetLogs.php +++ b/app/Mcp/Tools/GetLogs.php @@ -6,6 +6,7 @@ use App\Mcp\Concerns\BuildsResponse; use App\Mcp\Concerns\ResolvesResource; use App\Mcp\Concerns\ResolvesTeam; use App\Models\Application; +use App\Models\EnvironmentVariable; use App\Models\Service; use App\Models\ServiceApplication; use App\Models\ServiceDatabase; @@ -65,14 +66,17 @@ class GetLogs extends Tool return $this->mcpSuccess($request, $this->respond($failure), ['resource_uuid' => $uuid, 'outcome' => 'unavailable']); } + $secrets = $this->logRedactionSecrets($resource); + try { $logs = $this->redactLogText( - $this->fetchLogs($resource, $resourceType, $lines, $showTimestamps) + $this->fetchLogs($resource, $resourceType, $lines, $showTimestamps), + $secrets, ); } catch (MultipleContainersException $e) { $payload = $this->failurePayload( reason: 'multiple_containers', - message: $e->getMessage(), + message: $this->redactLogText($e->getMessage(), $secrets), resourceType: $resourceType, uuid: $uuid, status: $resource->status ?? null, @@ -85,7 +89,7 @@ class GetLogs extends Tool return $this->mcpSuccess($request, $this->respond( $this->failurePayload( reason: 'log_fetch_failed', - message: $e->getMessage(), + message: $this->redactLogText($e->getMessage(), $secrets), resourceType: $resourceType, uuid: $uuid, status: $resource->status ?? null, @@ -104,6 +108,20 @@ class GetLogs extends Tool ]), ['resource_uuid' => $uuid]); } + /** + * Environment values and database credentials of the resource that owns the log output. + * + * @return array{known: array, locked: array} + */ + private function logRedactionSecrets(mixed $resource): array + { + if ($resource instanceof ServiceApplication || $resource instanceof ServiceDatabase) { + $resource = $resource->service; + } + + return EnvironmentVariable::logRedactionSecretsFor($resource); + } + /** * @return array|null */ diff --git a/app/Models/Application.php b/app/Models/Application.php index 8d047aa68b..2b14b3c86b 100644 --- a/app/Models/Application.php +++ b/app/Models/Application.php @@ -748,6 +748,14 @@ class Application extends BaseModel return $this->hasOne(ApplicationSetting::class); } + /** + * Whether logs hide every environment value, or only locked values. + */ + public function redactsAllEnvValuesInLogs(): bool + { + return $this->settings?->redact_all_env_values_in_logs ?? true; + } + public function persistentStorages() { return $this->morphMany(LocalPersistentVolume::class, 'resource'); diff --git a/app/Models/ApplicationDeploymentQueue.php b/app/Models/ApplicationDeploymentQueue.php index 5994f187de..2979cab170 100644 --- a/app/Models/ApplicationDeploymentQueue.php +++ b/app/Models/ApplicationDeploymentQueue.php @@ -4,10 +4,12 @@ namespace App\Models; use App\Casts\EncryptedArrayCast; use App\Enums\ApplicationDeploymentStatus; +use App\Services\Security\SensitiveDataRedactor; use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Model; use Illuminate\Support\Carbon; use Illuminate\Support\Facades\DB; +use Illuminate\Support\Facades\Log; use OpenApi\Attributes as OA; #[OA\Schema( @@ -47,12 +49,21 @@ use OpenApi\Attributes as OA; class ApplicationDeploymentQueue extends Model { /** - * Kept in memory only, never saved. + * Log redaction variants of remote secrets. Kept in memory only, never saved. * - * @var array + * @var array */ private array $remoteSecretsForRedaction = []; + public const HIDDEN_ENV_VALUES_HINT = 'Some environment variable values are hidden in this log. To show values that are not locked, go to Environment Variables > Environment values in logs and select "Hide locked values only".'; + + /** @var array{known: array, locked: array}|null */ + private ?array $logRedactionSecrets = null; + + private bool $hiddenEnvValuesHintAdded = false; + + private bool $corruptedLogsWarned = false; + protected static function booted(): void { static::created(function (ApplicationDeploymentQueue $deployment): void { @@ -204,58 +215,49 @@ class ApplicationDeploymentQueue extends Model */ public function redactRemoteSecrets(array $secrets): void { - $this->remoteSecretsForRedaction = $secrets; + $this->remoteSecretsForRedaction = EnvironmentVariable::remoteSecretLogRedactionValues($secrets); } - private function redactSensitiveInfo($text) + /** + * Redact a deployment log text. The application secrets are collected once per deployment. + */ + public function redactLogText(string $text): string { try { - $text = remove_iip($text); - - $app = $this->application; - if (! $app) { - return $text; - } - - $lockedVars = collect([]); - - if ($app->environment_variables) { - $lockedVars = $lockedVars->merge( - $app->environment_variables - ->where('is_shown_once', true) - ->flatMap(fn (EnvironmentVariable $variable): array => $variable->logRedactionValues()) - ->filter() - ); - } - - if ($this->pull_request_id !== 0 && $app->environment_variables_preview) { - $lockedVars = $lockedVars->merge( - $app->environment_variables_preview - ->where('is_shown_once', true) - ->flatMap(fn (EnvironmentVariable $variable): array => $variable->logRedactionValues()) - ->filter() - ); - } - - $lockedVars = $lockedVars->merge(EnvironmentVariable::remoteSecretLogRedactionValues($this->remoteSecretsForRedaction)); - - foreach ($lockedVars as $key => $value) { - $escapedValue = preg_quote($value, '/'); - $text = preg_replace( - '/'.$escapedValue.'/', - REDACTED, - $text - ); - } - - return is_string($text) ? $text : REDACTED; + // The deployment job registers its remote secrets with redactRemoteSecrets(), so they are not fetched again. + $this->logRedactionSecrets ??= EnvironmentVariable::logRedactionSecretsFor($this->application, $this->pull_request_id !== 0, resolveRemoteSecrets: false); } catch (\Throwable) { return REDACTED; } + + return app(SensitiveDataRedactor::class)->redactText( + $text, + $this->logRedactionSecrets['known'], + array_merge($this->logRedactionSecrets['locked'], $this->remoteSecretsForRedaction), + ); } - public function addLogEntry(string $message, string $type = 'stdout', bool $hidden = false) + /** + * Tell if the "redact all values" setting hides an environment value that is not locked in this text. + */ + private function hidesUnlockedEnvValues(string $text): bool { + if ($this->logRedactionSecrets === null) { + return false; + } + + $unlockedValues = array_diff($this->logRedactionSecrets['known'], $this->logRedactionSecrets['locked'], $this->remoteSecretsForRedaction); + + return app(SensitiveDataRedactor::class)->containsKnownSecret($text, $unlockedValues); + } + + public function addLogEntry( + string $message, + string $type = 'stdout', + bool $hidden = false, + ?string $command = null, + int $batch = 1, + ): void { if ($type === 'error') { $type = 'stderr'; } @@ -264,30 +266,62 @@ class ApplicationDeploymentQueue extends Model $message = "\n".$message; } $newLogEntry = [ - 'command' => null, - 'output' => $this->redactSensitiveInfo($message), + 'command' => $command === null ? null : $this->redactLogText($command), + 'output' => $this->redactLogText($message), 'type' => $type, 'timestamp' => Carbon::now('UTC'), 'hidden' => $hidden, - 'batch' => 1, + 'batch' => $batch, ]; - // Use a transaction to ensure atomicity - DB::transaction(function () use ($newLogEntry) { - // Reload the model to get the latest logs - $this->refresh(); + $addHiddenEnvValuesHint = ! $this->hiddenEnvValuesHintAdded && $this->hidesUnlockedEnvValues($message.' '.$command); - if ($this->logs) { - $previousLogs = json_decode($this->logs, associative: true, flags: JSON_THROW_ON_ERROR); - $newLogEntry['order'] = count($previousLogs) + 1; - $previousLogs[] = $newLogEntry; - $this->logs = json_encode($previousLogs, flags: JSON_THROW_ON_ERROR); - } else { - $this->logs = json_encode([$newLogEntry], flags: JSON_THROW_ON_ERROR); + DB::transaction(function () use ($newLogEntry, $addHiddenEnvValuesHint): void { + $storedLogs = static::query()->whereKey($this->getKey())->lockForUpdate()->value('logs'); + $previousLogs = $this->decodeLogs($storedLogs); + if ($previousLogs === null) { + // Keep the stored log history instead of overwriting it with a new list. + if (! $this->corruptedLogsWarned) { + $this->corruptedLogsWarned = true; + Log::warning('Skipped a deployment log entry because the stored deployment logs cannot be decoded.', [ + 'deployment_uuid' => $this->deployment_uuid, + ]); + } + + return; } + $newLogEntry['order'] = count($previousLogs) + 1; + $previousLogs[] = $newLogEntry; + if ($addHiddenEnvValuesHint) { + $this->hiddenEnvValuesHintAdded = true; + if (! collect($previousLogs)->contains('output', self::HIDDEN_ENV_VALUES_HINT)) { + $previousLogs[] = [...$newLogEntry, 'command' => null, 'output' => self::HIDDEN_ENV_VALUES_HINT, 'type' => 'stdout', 'hidden' => false, 'order' => $newLogEntry['order'] + 1]; + } + } + $logs = json_encode($previousLogs, flags: JSON_THROW_ON_ERROR | JSON_INVALID_UTF8_SUBSTITUTE); - // Save without triggering events to prevent potential race conditions - $this->saveQuietly(); + // Update only the logs column without model events to prevent race conditions. + static::query()->whereKey($this->getKey())->update(['logs' => $logs]); + $this->logs = $logs; + $this->syncOriginalAttribute('logs'); }); } + + /** + * @return array|null Null when the stored logs are not blank and cannot be decoded. + */ + private function decodeLogs(?string $logs): ?array + { + if (blank($logs)) { + return []; + } + + try { + $decoded = json_decode($logs, associative: true, flags: JSON_THROW_ON_ERROR); + } catch (\JsonException) { + return null; + } + + return is_array($decoded) ? array_values($decoded) : null; + } } diff --git a/app/Models/ApplicationSetting.php b/app/Models/ApplicationSetting.php index 2e45fa8788..41249a5201 100644 --- a/app/Models/ApplicationSetting.php +++ b/app/Models/ApplicationSetting.php @@ -43,6 +43,7 @@ use OpenApi\Attributes as OA; 'is_git_shallow_clone_enabled' => ['type' => 'boolean'], 'is_pr_deployments_public_enabled' => ['type' => 'boolean'], 'use_build_secrets' => ['type' => 'boolean'], + 'redact_all_env_values_in_logs' => ['type' => 'boolean', 'description' => 'Hide every environment variable value in logs. When false, only locked values are hidden.'], 'inject_build_args_to_dockerfile' => ['type' => 'boolean'], 'include_source_commit_in_build' => ['type' => 'boolean'], 'docker_images_to_keep' => ['type' => 'integer'], @@ -87,6 +88,7 @@ class ApplicationSetting extends Model 'is_stripprefix_enabled' => 'boolean', 'connect_to_docker_network' => 'boolean', 'is_env_sorting_enabled' => 'boolean', + 'redact_all_env_values_in_logs' => 'boolean', 'disable_build_cache' => 'boolean', ]; @@ -125,6 +127,7 @@ class ApplicationSetting extends Model 'is_git_shallow_clone_enabled', 'is_pr_deployments_public_enabled', 'use_build_secrets', + 'redact_all_env_values_in_logs', 'inject_build_args_to_dockerfile', 'include_source_commit_in_build', 'docker_images_to_keep', diff --git a/app/Models/EnvironmentVariable.php b/app/Models/EnvironmentVariable.php index 050fc15342..acb4ca139f 100644 --- a/app/Models/EnvironmentVariable.php +++ b/app/Models/EnvironmentVariable.php @@ -2,12 +2,16 @@ namespace App\Models; +use App\Exceptions\RemoteSecretException; use App\Models\EnvironmentVariable as ModelsEnvironmentVariable; +use App\Support\RemoteSecretReferences; use App\Support\RemoteSecretValueFormatter; use App\Support\ValidationPatterns; use App\Traits\Auditable; use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Casts\Attribute; +use Illuminate\Support\Collection; +use Illuminate\Support\Facades\Log; use OpenApi\Attributes as OA; #[OA\Schema( @@ -215,7 +219,10 @@ class EnvironmentVariable extends BaseModel return $this->resolveRealValue(revealLockedSharedVariables: false); } - private function resolveRealValue(bool $revealLockedSharedVariables = true, bool $escapeLegacyValue = true): ?string + /** + * @param Collection|null $sharedVariables + */ + private function resolveRealValue(bool $revealLockedSharedVariables = true, bool $escapeLegacyValue = true, ?Collection $sharedVariables = null): ?string { if (! $this->relationLoaded('resourceable')) { $this->load('resourceable'); @@ -236,7 +243,7 @@ class EnvironmentVariable extends BaseModel $resource->load('destination.server'); } - $value = $this->get_real_environment_variables_internal($this->value, $resource, null, $revealLockedSharedVariables); + $value = $this->get_real_environment_variables_internal($this->value, $resource, null, $revealLockedSharedVariables, $sharedVariables); return $value === null || ! $escapeLegacyValue || ! $this->uses_legacy_escaping ? $value : $this->legacyEscapedValue($value); } @@ -344,9 +351,12 @@ class EnvironmentVariable extends BaseModel } /** @return array */ - public function logRedactionValues(): array + /** + * @param Collection|null $sharedVariables Preloaded shared variables, to avoid one query per reference. + */ + public function logRedactionValues(?Collection $sharedVariables = null): array { - $resolvedValue = $this->resolveRealValue(escapeLegacyValue: false); + $resolvedValue = $this->resolveRealValue(escapeLegacyValue: false, sharedVariables: $sharedVariables); if (! is_string($resolvedValue) || $resolvedValue === '') { return []; } @@ -402,6 +412,177 @@ class EnvironmentVariable extends BaseModel return $values; } + /** + * Collect the values to hide in a resource's logs. + * + * Locked values, database passwords, and resolved remote secrets are always hidden, + * whatever their length. Other values are hidden unless the resource limits log redaction + * to locked values only. + * + * Remote secrets are fetched at most once per call, and only when a variable references one. + * Callers that already hold the fetched values, such as deployments, pass $resolveRemoteSecrets = false. + * + * @return array{known: array, locked: array} + */ + public static function logRedactionSecretsFor(Application|Service|StandalonePostgresql|StandaloneRedis|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse|StandaloneSqlite|null $resource, bool $includePreview = false, bool $resolveRemoteSecrets = true): array + { + if ($resource === null) { + return ['known' => [], 'locked' => []]; + } + + $isDatabase = ! $resource instanceof Application && ! $resource instanceof Service; + $secrets = ['known' => [], 'locked' => []]; + if ($isDatabase) { + foreach (array_keys($resource->getAttributes()) as $attribute) { + if (str_contains($attribute, 'password')) { + $value = $resource->getAttribute($attribute); + if (is_string($value) && $value !== '') { + $secrets['locked'][] = $value; + } + } + } + } + + $variables = collect($isDatabase ? $resource->runtime_environment_variables : $resource->environment_variables); + if ($includePreview && $resource instanceof Application) { + $variables = $variables->merge($resource->environment_variables_preview ?? []); + } + if ($variables->isEmpty()) { + return $secrets; + } + + // Load the resource relations once and share them, so each variable does not load its own copy. + $resource->loadMissing('environment.project.team'); + if (method_exists($resource, 'server')) { + $resource->loadMissing('server'); + } + if (method_exists($resource, 'destination')) { + $resource->loadMissing('destination.server'); + } + // Use copies so the resource's own loaded variables do not get a reference back to the resource. + $variables = $variables->map(fn (self $variable): self => (clone $variable)->setRelation('resourceable', $resource)); + $sharedVariables = self::sharedVariablesReferencedBy($variables, $resource->team()?->id); + + $redactAllValues = $resource->redactsAllEnvValuesInLogs(); + $unresolvedKeys = []; + $remoteSecretFailure = null; + foreach ($variables as $variable) { + if ($resolveRemoteSecrets) { + $secrets['locked'] = array_merge( + $secrets['locked'], + self::remoteSecretLogRedactionValuesFor($resource, $variable, $sharedVariables, $remoteSecretFailure, $unresolvedKeys), + ); + } + if ($variable->isLockedForLogs($sharedVariables)) { + $secrets['locked'] = array_merge($secrets['locked'], $variable->logRedactionValues($sharedVariables)); + } elseif ($redactAllValues) { + $secrets['known'] = array_merge($secrets['known'], $variable->logRedactionValues($sharedVariables)); + } + } + + if ($remoteSecretFailure !== null) { + Log::warning('Could not resolve remote secrets for log redaction; their resolved values are not hidden.', [ + 'resource_type' => $resource->getMorphClass(), + 'resource_uuid' => $resource->uuid, + 'variables' => array_values(array_unique($unresolvedKeys)), + 'exception' => $remoteSecretFailure::class, + ]); + } + + return $secrets; + } + + /** + * Resolved remote secret values of a variable; these are always hidden. + * + * When the references cannot be resolved, export still works: the reference text and any keys + * that did resolve are hidden, and the variable is reported. After a failed or empty fetch the + * source is not asked again. + * + * @param Collection|null $sharedVariables + * @param array $unresolvedKeys + * @return array + */ + private static function remoteSecretLogRedactionValuesFor(Application|Service|StandalonePostgresql|StandaloneRedis|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse|StandaloneSqlite $resource, self $variable, ?Collection $sharedVariables, ?RemoteSecretException &$failure, array &$unresolvedKeys): array + { + // Cheap check first, so variables without references never query or contact a source. + $value = $variable->resolveRealValue(escapeLegacyValue: false, sharedVariables: $sharedVariables); + if (! RemoteSecretReferences::containsReference($value) || ! $resource->environmentVariableUsesSecretManager($variable)) { + return []; + } + + if ($failure === null || $resource->resolvedSecretManagerValuesForRedaction() !== []) { + try { + return self::remoteSecretLogRedactionValues([$resource->resolveSecretManagerEnvironmentVariableValue($variable)]); + } catch (RemoteSecretException $e) { + $failure = $e; + } + } + + $unresolvedKeys[] = $variable->key; + $available = array_intersect_key($resource->resolvedSecretManagerValuesForRedaction(), array_flip(RemoteSecretReferences::referencedKeys($value))); + + return [...self::remoteSecretLogRedactionValues(array_values($available)), ...$variable->logRedactionValues($sharedVariables)]; + } + + /** + * A variable is locked when it is locked itself or references a locked shared variable. + * + * @param Collection|null $sharedVariables Preloaded shared variables of the resource's team. + */ + public function isLockedForLogs(?Collection $sharedVariables = null): bool + { + if ($this->is_shown_once) { + return true; + } + + $references = self::sharedVariableReferences($this->value); + if ($references === []) { + return false; + } + + $sharedVariables ??= self::sharedVariablesReferencedBy(collect([$this]), $this->resourceable?->team()?->id); + if ($sharedVariables === null) { + return true; + } + + return $sharedVariables->contains(fn (SharedEnvironmentVariable $shared): bool => $shared->is_shown_once + && in_array(['type' => $shared->type, 'key' => $shared->key], $references, true)); + } + + /** + * Load every shared variable that the given variables reference, with one query. + * + * @param Collection $variables + * @return Collection|null Null when the team is unknown. + */ + private static function sharedVariablesReferencedBy(Collection $variables, ?int $teamId): ?Collection + { + if ($teamId === null) { + return null; + } + + $keys = $variables + ->flatMap(fn (self $variable): array => self::sharedVariableReferences($variable->value)) + ->pluck('key') + ->unique() + ->values(); + + return $keys->isEmpty() + ? collect() + : SharedEnvironmentVariable::query()->where('team_id', $teamId)->whereIn('key', $keys)->get(); + } + + /** + * @return array + */ + private static function sharedVariableReferences(?string $value): array + { + preg_match_all('/{{\s*(\w+)\.([^}\s]+)\s*}}/', (string) $value, $matches, PREG_SET_ORDER); + + return array_map(fn (array $match): array => ['type' => $match[1], 'key' => $match[2]], $matches); + } + public function resolveReferencedValue(): ?string { $value = $this->value; @@ -420,7 +601,10 @@ class EnvironmentVariable extends BaseModel ->first()?->value ?? $value; } - private function get_real_environment_variables_internal(?string $environment_variable = null, $resource = null, $serverOverride = null, bool $revealLockedSharedVariables = true) + /** + * @param Collection|null $sharedVariables + */ + private function get_real_environment_variables_internal(?string $environment_variable = null, $resource = null, $serverOverride = null, bool $revealLockedSharedVariables = true, ?Collection $sharedVariables = null) { if (is_null($environment_variable) || $environment_variable === '' || is_null($resource)) { return $environment_variable; @@ -455,11 +639,15 @@ class EnvironmentVariable extends BaseModel if (is_null($id)) { continue; } - $found = SharedEnvironmentVariable::where('type', $type) - ->where('key', $variable) - ->where('team_id', $resource->team()->id) - ->where("{$type}_id", $id) - ->first(); + $found = $sharedVariables !== null + ? $sharedVariables->first(fn (SharedEnvironmentVariable $shared): bool => $shared->type === $type->value() + && $shared->key === $variable->value() + && (int) $shared->{"{$type}_id"} === (int) $id) + : SharedEnvironmentVariable::where('type', $type) + ->where('key', $variable) + ->where('team_id', $resource->team()->id) + ->where("{$type}_id", $id) + ->first(); if ($found && ($revealLockedSharedVariables || ! $found->is_shown_once)) { $environment_variable = str($environment_variable)->replace("{{{$sharedEnv}}}", $found->value); } diff --git a/app/Models/Server.php b/app/Models/Server.php index e460a08294..e6b85e522f 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -375,7 +375,7 @@ class Server extends BaseModel public function setValidationLogsAttribute($value): void { $this->attributes['validation_logs'] = $value !== null - ? Purify::config('validation_logs')->clean($value) + ? Purify::config('validation_logs')->clean(remove_iip($value)) : null; } diff --git a/app/Models/Service.php b/app/Models/Service.php index 724768361c..200806a830 100644 --- a/app/Models/Service.php +++ b/app/Models/Service.php @@ -35,6 +35,7 @@ use OpenApi\Attributes as OA; 'destination_id' => ['type' => 'integer', 'description' => 'The unique identifier of the destination where the service is running.'], 'connect_to_docker_network' => ['type' => 'boolean', 'description' => 'The flag to connect the service to the predefined Docker network.'], 'is_container_label_escape_enabled' => ['type' => 'boolean', 'description' => 'The flag to enable the container label escape.'], + 'redact_all_env_values_in_logs' => ['type' => 'boolean', 'description' => 'Hide every environment variable value in logs. When false, only locked values are hidden.'], 'is_container_label_readonly_enabled' => ['type' => 'boolean', 'description' => 'The flag to enable the container label readonly.'], 'config_hash' => ['type' => 'string', 'description' => 'The hash of the service configuration.'], 'service_type' => ['type' => 'string', 'description' => 'The type of the service.'], @@ -60,6 +61,7 @@ class Service extends BaseModel 'config_hash', 'compose_parsing_version', 'is_container_label_escape_enabled', + 'redact_all_env_values_in_logs', 'environment_id', 'server_id', 'destination_id', @@ -68,6 +70,10 @@ class Service extends BaseModel protected $appends = ['server_status', 'status']; + protected $casts = [ + 'redact_all_env_values_in_logs' => 'boolean', + ]; + /** * Sensitive fields hidden by default in serialized output (toArray/toJson). * API controllers should call makeVisible([...]) for callers with the @@ -1599,6 +1605,14 @@ class Service extends BaseModel return $this->hasMany(ScheduledTask::class)->orderBy('name', 'asc'); } + /** + * Whether logs hide every environment value, or only locked values. + */ + public function redactsAllEnvValuesInLogs(): bool + { + return $this->redact_all_env_values_in_logs ?? true; + } + public function environment_variables() { return $this->morphMany(EnvironmentVariable::class, 'resourceable'); diff --git a/app/Models/StandaloneClickhouse.php b/app/Models/StandaloneClickhouse.php index 24455ffdd0..147dab9832 100644 --- a/app/Models/StandaloneClickhouse.php +++ b/app/Models/StandaloneClickhouse.php @@ -5,6 +5,7 @@ namespace App\Models; use App\Traits\Auditable; use App\Traits\ClearsGlobalSearchCache; use App\Traits\HasDatabaseHealthCheck; +use App\Traits\HasLogRedactionSetting; use App\Traits\HasMetrics; use App\Traits\HasSafeStringAttribute; use App\Traits\HasSecretManager; @@ -15,7 +16,7 @@ use Throwable; class StandaloneClickhouse extends BaseModel { - use Auditable, ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasSafeStringAttribute, HasSecretManager, SoftDeletes; + use Auditable, ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasLogRedactionSetting, HasMetrics, HasSafeStringAttribute, HasSecretManager, SoftDeletes; protected array $auditExclude = ['last_online_at']; diff --git a/app/Models/StandaloneDragonfly.php b/app/Models/StandaloneDragonfly.php index 2f03815b25..7de46038c6 100644 --- a/app/Models/StandaloneDragonfly.php +++ b/app/Models/StandaloneDragonfly.php @@ -5,6 +5,7 @@ namespace App\Models; use App\Traits\Auditable; use App\Traits\ClearsGlobalSearchCache; use App\Traits\HasDatabaseHealthCheck; +use App\Traits\HasLogRedactionSetting; use App\Traits\HasMetrics; use App\Traits\HasSafeStringAttribute; use App\Traits\HasSecretManager; @@ -14,7 +15,7 @@ use Illuminate\Database\Eloquent\SoftDeletes; class StandaloneDragonfly extends BaseModel { - use Auditable, ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasSafeStringAttribute, HasSecretManager, SoftDeletes; + use Auditable, ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasLogRedactionSetting, HasMetrics, HasSafeStringAttribute, HasSecretManager, SoftDeletes; protected $fillable = [ 'uuid', diff --git a/app/Models/StandaloneKeydb.php b/app/Models/StandaloneKeydb.php index ca2c4174c1..3257a9c108 100644 --- a/app/Models/StandaloneKeydb.php +++ b/app/Models/StandaloneKeydb.php @@ -5,6 +5,7 @@ namespace App\Models; use App\Traits\Auditable; use App\Traits\ClearsGlobalSearchCache; use App\Traits\HasDatabaseHealthCheck; +use App\Traits\HasLogRedactionSetting; use App\Traits\HasMetrics; use App\Traits\HasSafeStringAttribute; use App\Traits\HasSecretManager; @@ -14,7 +15,7 @@ use Illuminate\Database\Eloquent\SoftDeletes; class StandaloneKeydb extends BaseModel { - use Auditable, ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasSafeStringAttribute, HasSecretManager, SoftDeletes; + use Auditable, ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasLogRedactionSetting, HasMetrics, HasSafeStringAttribute, HasSecretManager, SoftDeletes; protected $fillable = [ 'uuid', diff --git a/app/Models/StandaloneMariadb.php b/app/Models/StandaloneMariadb.php index 05aac1b42d..ea3bc39ce4 100644 --- a/app/Models/StandaloneMariadb.php +++ b/app/Models/StandaloneMariadb.php @@ -5,6 +5,7 @@ namespace App\Models; use App\Traits\Auditable; use App\Traits\ClearsGlobalSearchCache; use App\Traits\HasDatabaseHealthCheck; +use App\Traits\HasLogRedactionSetting; use App\Traits\HasMetrics; use App\Traits\HasSafeStringAttribute; use App\Traits\HasSecretManager; @@ -15,7 +16,7 @@ use Illuminate\Database\Eloquent\SoftDeletes; class StandaloneMariadb extends BaseModel { - use Auditable, ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasSafeStringAttribute, HasSecretManager, SoftDeletes; + use Auditable, ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasLogRedactionSetting, HasMetrics, HasSafeStringAttribute, HasSecretManager, SoftDeletes; protected $fillable = [ 'uuid', diff --git a/app/Models/StandaloneMongodb.php b/app/Models/StandaloneMongodb.php index a273132871..41bf67678b 100644 --- a/app/Models/StandaloneMongodb.php +++ b/app/Models/StandaloneMongodb.php @@ -5,6 +5,7 @@ namespace App\Models; use App\Traits\Auditable; use App\Traits\ClearsGlobalSearchCache; use App\Traits\HasDatabaseHealthCheck; +use App\Traits\HasLogRedactionSetting; use App\Traits\HasMetrics; use App\Traits\HasSafeStringAttribute; use App\Traits\HasSecretManager; @@ -14,7 +15,7 @@ use Illuminate\Database\Eloquent\SoftDeletes; class StandaloneMongodb extends BaseModel { - use Auditable, ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasSafeStringAttribute, HasSecretManager, SoftDeletes; + use Auditable, ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasLogRedactionSetting, HasMetrics, HasSafeStringAttribute, HasSecretManager, SoftDeletes; protected $fillable = [ 'uuid', diff --git a/app/Models/StandaloneMysql.php b/app/Models/StandaloneMysql.php index 7475421ba1..2593bafa32 100644 --- a/app/Models/StandaloneMysql.php +++ b/app/Models/StandaloneMysql.php @@ -5,6 +5,7 @@ namespace App\Models; use App\Traits\Auditable; use App\Traits\ClearsGlobalSearchCache; use App\Traits\HasDatabaseHealthCheck; +use App\Traits\HasLogRedactionSetting; use App\Traits\HasMetrics; use App\Traits\HasSafeStringAttribute; use App\Traits\HasSecretManager; @@ -14,7 +15,7 @@ use Illuminate\Database\Eloquent\SoftDeletes; class StandaloneMysql extends BaseModel { - use Auditable, ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasSafeStringAttribute, HasSecretManager, SoftDeletes; + use Auditable, ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasLogRedactionSetting, HasMetrics, HasSafeStringAttribute, HasSecretManager, SoftDeletes; protected $fillable = [ 'uuid', diff --git a/app/Models/StandalonePostgresql.php b/app/Models/StandalonePostgresql.php index 1cb5c4ec41..9e6f66c958 100644 --- a/app/Models/StandalonePostgresql.php +++ b/app/Models/StandalonePostgresql.php @@ -5,6 +5,7 @@ namespace App\Models; use App\Traits\Auditable; use App\Traits\ClearsGlobalSearchCache; use App\Traits\HasDatabaseHealthCheck; +use App\Traits\HasLogRedactionSetting; use App\Traits\HasMetrics; use App\Traits\HasSafeStringAttribute; use App\Traits\HasSecretManager; @@ -14,7 +15,7 @@ use Illuminate\Database\Eloquent\SoftDeletes; class StandalonePostgresql extends BaseModel { - use Auditable, ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasSafeStringAttribute, HasSecretManager, SoftDeletes; + use Auditable, ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasLogRedactionSetting, HasMetrics, HasSafeStringAttribute, HasSecretManager, SoftDeletes; protected $fillable = [ 'uuid', diff --git a/app/Models/StandaloneRedis.php b/app/Models/StandaloneRedis.php index abc1d4c0c6..7fe86c9974 100644 --- a/app/Models/StandaloneRedis.php +++ b/app/Models/StandaloneRedis.php @@ -5,6 +5,7 @@ namespace App\Models; use App\Traits\Auditable; use App\Traits\ClearsGlobalSearchCache; use App\Traits\HasDatabaseHealthCheck; +use App\Traits\HasLogRedactionSetting; use App\Traits\HasMetrics; use App\Traits\HasSafeStringAttribute; use App\Traits\HasSecretManager; @@ -14,7 +15,7 @@ use Illuminate\Database\Eloquent\SoftDeletes; class StandaloneRedis extends BaseModel { - use Auditable, ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasSafeStringAttribute, HasSecretManager, SoftDeletes; + use Auditable, ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasLogRedactionSetting, HasMetrics, HasSafeStringAttribute, HasSecretManager, SoftDeletes; protected array $auditExclude = ['last_online_at']; diff --git a/app/Models/StandaloneSqlite.php b/app/Models/StandaloneSqlite.php index 4f5c1cfc74..e3490c1031 100644 --- a/app/Models/StandaloneSqlite.php +++ b/app/Models/StandaloneSqlite.php @@ -5,6 +5,7 @@ namespace App\Models; use App\Traits\Auditable; use App\Traits\ClearsGlobalSearchCache; use App\Traits\HasDatabaseHealthCheck; +use App\Traits\HasLogRedactionSetting; use App\Traits\HasMetrics; use App\Traits\HasSafeStringAttribute; use App\Traits\HasSecretManager; @@ -16,7 +17,7 @@ use InvalidArgumentException; class StandaloneSqlite extends BaseModel { - use Auditable, ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasSafeStringAttribute, HasSecretManager, SoftDeletes; + use Auditable, ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasLogRedactionSetting, HasMetrics, HasSafeStringAttribute, HasSecretManager, SoftDeletes; public const DATA_DIRECTORY = '/var/lib/sqlite'; diff --git a/app/Notifications/Channels/EmailChannel.php b/app/Notifications/Channels/EmailChannel.php index 9a14da58a1..07853f139e 100644 --- a/app/Notifications/Channels/EmailChannel.php +++ b/app/Notifications/Channels/EmailChannel.php @@ -5,6 +5,7 @@ namespace App\Notifications\Channels; use App\Exceptions\NonReportableException; use App\Models\EmailNotificationSettings; use App\Models\Team; +use App\Services\Security\SensitiveDataRedactor; use App\Support\SmtpTransportFactory; use Exception; use Illuminate\Notifications\Notification; @@ -151,9 +152,9 @@ class EmailChannel send_internal_notification(sprintf( "Resend Error\nStatus Code: %s\nMessage: %s\nNotification: %s\nEmail Settings:\n%s", $statusCode, - $e->getErrorMessage(), + remove_iip($e->getErrorMessage()), get_class($notification), - json_encode($emailSettings, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES) + json_encode(app(SensitiveDataRedactor::class)->redactValue($emailSettings), JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES) )); // Don't report expected errors (invalid keys, validation) to Sentry @@ -172,7 +173,7 @@ class EmailChannel throw $e; } catch (TransporterException $e) { - send_internal_notification("Resend Transport Error: {$e->getMessage()}"); + send_internal_notification('Resend Transport Error: '.remove_iip($e->getMessage())); throw new Exception('Unable to connect to Resend API. Please check your internet connection and try again.'); } catch (\Throwable $e) { // Check if this is a Resend domain verification error on cloud instances diff --git a/app/Services/DatabaseStartCommandExecutor.php b/app/Services/DatabaseStartCommandExecutor.php index 96e2f500bd..4fc38a9539 100644 --- a/app/Services/DatabaseStartCommandExecutor.php +++ b/app/Services/DatabaseStartCommandExecutor.php @@ -4,9 +4,12 @@ namespace App\Services; use App\Enums\ProcessStatus; use App\Helpers\SshMultiplexingHelper; +use App\Services\Security\SensitiveDataRedactor; use Illuminate\Database\Eloquent\Model; +use Illuminate\Process\Exceptions\ProcessTimedOutException; use Illuminate\Support\Facades\Process; use Spatie\Activitylog\Models\Activity; +use Symfony\Component\Process\Exception\ProcessTimedOutException as SymfonyProcessTimedOutException; class DatabaseStartCommandExecutor { @@ -25,13 +28,31 @@ class DatabaseStartCommandExecutor $activity->properties = $activity->properties->merge(['status' => ProcessStatus::IN_PROGRESS->value]); $activity->save(); - $process = Process::timeout(config('constants.ssh.command_timeout')) - ->idleTimeout(3600) - ->start($remoteCommand, function (string $type, string $output) use ($activity, $secrets): void { - $this->appendOutput($activity, $type, $this->redact($output, $secrets)); - }); + $timeout = config('constants.ssh.command_timeout'); + // Output is redacted per complete line, so a secret split across two output chunks is still found. + $pending = ['stdout' => '', 'stderr' => '']; + try { + $process = Process::timeout($timeout) + ->idleTimeout(3600) + ->start($remoteCommand, function (string $type, string $output) use ($activity, $secrets, &$pending): void { + $pending[$type] = ($pending[$type] ?? '').$output; + $end = max(strrpos($pending[$type], "\n"), strrpos($pending[$type], "\r")); + if ($end === false) { + return; + } + $this->appendOutput($activity, $type, $this->redact(substr($pending[$type], 0, $end + 1), $secrets)); + $pending[$type] = substr($pending[$type], $end + 1); + }); - $result = $process->wait(); + $result = $process->wait(); + } catch (ProcessTimedOutException|SymfonyProcessTimedOutException) { + // The timeout message contains the full SSH command, which can contain secrets. + throw new \RuntimeException("The command on the server timed out after {$timeout} seconds.", 124); + } finally { + foreach ($pending as $type => $output) { + $this->appendOutput($activity, $type, $this->redact($output, $secrets)); + } + } $status = $result->successful() ? ProcessStatus::FINISHED : ProcessStatus::ERROR; $activity->properties = $activity->properties->merge([ 'status' => $status->value, @@ -46,15 +67,12 @@ class DatabaseStartCommandExecutor return $activity; } + /** + * @param array $secrets + */ private function redact(string $value, array $secrets): string { - foreach ($secrets as $secret) { - if (is_string($secret) && $secret !== '') { - $value = str_replace($secret, REDACTED, $value); - } - } - - return sanitize_utf8_text(remove_iip($value)); + return sanitize_utf8_text(resolve(SensitiveDataRedactor::class)->redactText($value, lockedSecrets: $secrets)); } private function appendOutput(Activity $activity, string $type, string $output): void diff --git a/app/Services/DockerRegistryLogins.php b/app/Services/DockerRegistryLogins.php index a165df7e7b..8406fa9ec4 100644 --- a/app/Services/DockerRegistryLogins.php +++ b/app/Services/DockerRegistryLogins.php @@ -6,10 +6,12 @@ use App\Helpers\SshMultiplexingHelper; use App\Models\Application; use App\Models\Server; use App\Models\Service; +use Illuminate\Process\Exceptions\ProcessTimedOutException; use Illuminate\Support\Arr; use Illuminate\Support\Collection; use Illuminate\Support\Facades\Process; use RuntimeException; +use Symfony\Component\Process\Exception\ProcessTimedOutException as SymfonyProcessTimedOutException; class DockerRegistryLogins { @@ -274,7 +276,12 @@ class DockerRegistryLogins private static function run(Server $server, string $command, array $secrets = [], ?string $emptyErrorMessage = null): void { $commands = $server->isNonRoot() ? parseCommandsByLineForSudo(collect([$command]), $server) : [$command]; - $process = Process::timeout(60)->run(SshMultiplexingHelper::generateSshCommand($server, implode("\n", $commands))); + try { + $process = Process::timeout(60)->run(SshMultiplexingHelper::generateSshCommand($server, implode("\n", $commands))); + } catch (ProcessTimedOutException|SymfonyProcessTimedOutException) { + // The timeout message contains the full SSH command, including the registry password. + throw new RuntimeException('The command on the server timed out after 60 seconds.', 124); + } if ($process->exitCode() === 0) { return; diff --git a/app/Services/Security/SensitiveDataRedactor.php b/app/Services/Security/SensitiveDataRedactor.php new file mode 100644 index 0000000000..4ea688d1c6 --- /dev/null +++ b/app/Services/Security/SensitiveDataRedactor.php @@ -0,0 +1,325 @@ +redactEmails = false; + + return $redactor; + } + + /** + * Tell if redactText() would hide one of these known secrets in the text. + * + * @param array $knownSecrets + */ + public function containsKnownSecret(string $text, array $knownSecrets): bool + { + foreach ($knownSecrets as $secret) { + if (is_scalar($secret) && trim((string) $secret) !== '' && strlen((string) $secret) >= self::MIN_KNOWN_SECRET_LENGTH && str_contains($text, (string) $secret)) { + return true; + } + } + + return false; + } + + /** + * @param array $knownSecrets Exact values to replace when they are long enough to be safe to match, such as unlocked environment values. + * @param array $lockedSecrets Exact values to always replace, whatever their length, such as locked or remote secrets. + */ + public function redactText(string $text, array $knownSecrets = [], array $lockedSecrets = []): string + { + try { + $text = $this->normalizeText($text); + $text = $this->replaceKnownSecrets($text, $knownSecrets, $lockedSecrets); + + $redacted = ''; + foreach ($this->chunks($text) as $chunk) { + $redacted .= $this->redactChunk($chunk); + } + + return $redacted; + } catch (Throwable) { + return self::FAILURE_MESSAGE; + } + } + + /** + * @param array $knownSecrets + */ + public function redactValue(mixed $value, array $knownSecrets = [], ?string $key = null): mixed + { + try { + if ($key !== null && $this->isSensitiveKey($key)) { + return REDACTED; + } + if (is_array($value)) { + $redacted = []; + foreach ($value as $itemKey => $itemValue) { + $redacted[$itemKey] = $this->redactValue($itemValue, $knownSecrets, is_string($itemKey) ? $itemKey : null); + } + + return $redacted; + } + if (is_string($value)) { + return $this->redactText($value, $knownSecrets); + } + if (! is_object($value) || $value instanceof UnitEnum || $value instanceof DateTimeInterface) { + return $value; + } + if ($value instanceof Arrayable) { + return $this->redactValue($value->toArray(), $knownSecrets); + } + if ($value instanceof JsonSerializable) { + return $this->redactValue($value->jsonSerialize(), $knownSecrets); + } + if ($value instanceof Stringable) { + return $this->redactText((string) $value, $knownSecrets); + } + + return '[object '.$value::class.']'; + } catch (Throwable) { + return self::FAILURE_MESSAGE; + } + } + + /** + * Redact one chunk. When a pattern hits a PCRE limit, redact the chunk line by line so one huge line + * or an unterminated private key does not suppress the whole log. A line that still fails is suppressed. + */ + private function redactChunk(string $chunk): string + { + try { + return $this->redactAssignments($this->redactPatterns($chunk)); + } catch (RuntimeException) { + $redacted = ''; + $insidePrivateKey = false; + foreach (preg_split('/(?<=\n)/', $chunk) ?: [$chunk] as $line) { + if (! $insidePrivateKey && preg_match('/-----BEGIN [A-Z ]*PRIVATE KEY-----/', $line) === 1 && preg_match('/-----END [A-Z ]*PRIVATE KEY-----/', $line) !== 1) { + $insidePrivateKey = true; + $redacted .= REDACTED.(str_ends_with($line, "\n") ? "\n" : ''); + + continue; + } + if ($insidePrivateKey) { + $insidePrivateKey = preg_match('/-----END [A-Z ]*PRIVATE KEY-----/', $line) !== 1; + + continue; + } + try { + $redacted .= $this->redactAssignments($this->redactPatterns($line)); + } catch (RuntimeException) { + $redacted .= self::FAILURE_MESSAGE.(str_ends_with($line, "\n") ? "\n" : ''); + } + } + + return $redacted; + } + } + + private function normalizeText(string $text): string + { + $text = sanitize_utf8_text($text); + $text = preg_replace('/\x1B(?:[@-Z\\-_]|\[[0-?]*[ -\/]*[@-~])/', '', $text); + if ($text === null) { + throw new RuntimeException('Log normalization failed.'); + } + $text = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/', '', $text); + if ($text === null) { + throw new RuntimeException('Log normalization failed.'); + } + + return $text; + } + + /** + * Split large text on line boundaries so the regular expressions stay inside PCRE limits. + * A chunk never ends inside a private key block. + * + * @return iterable + */ + private function chunks(string $text): iterable + { + $length = strlen($text); + $offset = 0; + while ($offset < $length) { + $end = $this->lineEnd($text, $offset + self::CHUNK_BYTES); + $chunk = substr($text, $offset, $end - $offset); + if (preg_match('/-----BEGIN [A-Z ]*PRIVATE KEY-----(?![\s\S]*-----END [A-Z ]*PRIVATE KEY-----)/', $chunk) === 1) { + $end = preg_match('/-----END [A-Z ]*PRIVATE KEY-----/', $text, $match, PREG_OFFSET_CAPTURE, $end) === 1 + ? $this->lineEnd($text, $match[0][1] + strlen($match[0][0])) + : $length; + $chunk = substr($text, $offset, $end - $offset); + } + + yield $chunk; + $offset = $end; + } + } + + /** + * Find the end of the line that contains $position, without copying the text. + */ + private function lineEnd(string $text, int $position): int + { + if ($position >= strlen($text)) { + return strlen($text); + } + + $newline = strpos($text, "\n", $position); + + return $newline === false ? strlen($text) : $newline + 1; + } + + /** + * @param array $knownSecrets + * @param array $lockedSecrets + */ + private function replaceKnownSecrets(string $text, array $knownSecrets, array $lockedSecrets): string + { + $variants = []; + $shortVariants = []; + foreach ([[$knownSecrets, self::MIN_KNOWN_SECRET_LENGTH], [$lockedSecrets, 1]] as [$secrets, $minimumLength]) { + foreach ($secrets as $secret) { + if (! is_scalar($secret) || trim((string) $secret) === '' || strlen((string) $secret) < $minimumLength) { + continue; + } + $secret = (string) $secret; + $json = json_encode($secret, JSON_THROW_ON_ERROR | JSON_INVALID_UTF8_SUBSTITUTE); + // Short locked values such as "ab" would hit inside ordinary words, so they only match as whole tokens. + if (strlen($secret) < self::MIN_KNOWN_SECRET_LENGTH) { + array_push($shortVariants, $secret, rawurlencode($secret), substr($json, 1, -1)); + } else { + array_push($variants, $secret, rawurlencode($secret), substr($json, 1, -1)); + } + } + } + $byLengthDescending = fn (string $left, string $right): int => strlen($right) <=> strlen($left); + $variants = array_values(array_unique(array_filter($variants))); + usort($variants, $byLengthDescending); + $text = str_replace($variants, REDACTED, $text); + + // The text is valid UTF-8 after normalization, so a variant that is not valid UTF-8 cannot appear in it. + $shortVariants = array_values(array_unique(array_filter($shortVariants, fn (string $variant): bool => $variant !== '' && mb_check_encoding($variant, 'UTF-8')))); + if ($shortVariants === []) { + return $text; + } + usort($shortVariants, $byLengthDescending); + $alternatives = implode('|', array_map(fn (string $variant): string => preg_quote($variant, '/'), $shortVariants)); + $text = preg_replace('/(? preg_quote($key, '/'), $assignmentKeys)); + // A key matches when it is a sensitive name or ends with one, such as smtp_password or github-token. + $keys = '(?i:(?:[a-z0-9]+[_-])*(?:'.$keys.'))|[A-Z][A-Z0-9_]*(?:SECRET|PASSWORD|TOKEN|PASSWD|API_KEY|PRIVATE_KEY)[A-Z0-9_]*'; + $value = '(?"(?:[^"\\\\\r\n]|\\\\.)*"|\'(?:[^\'\\\\\r\n]|\\\\.)*\'|[^\s,;}\]]+)'; + // Skip values that an earlier rule already replaced, such as the token in https://x-access-token:@host. + $alreadyRedacted = '(?!'.preg_quote(REDACTED, '/').')'; + // Set when nothing but a delimiter follows the value on its line, as in YAML or object literals. + $endsValue = '(?(?=[^\S\r\n]*(?:$|[\r\n,;}\]])))?'; + // A quote that opens before the key and closes after the value wraps the whole assignment, as in + // the shell argument 'DB_PASSWORD=a b'. Hide everything up to the closing quote and keep the quote. + $text = preg_replace('/(?["\']?(?'.$keys.')["\']?\s*(?[=:])\s*)'.$alreadyRedacted.$value.$endsValue.'/m', function (array $match): string { + $value = $match['value']; + $quote = strlen($value) > 1 && in_array($value[0], ['"', "'"], true) && str_ends_with($value, $value[0]) ? $value[0] : ''; + // Prose such as "Failed to refresh token: Connection refused" uses a plain word key, a colon, and a plain word value. + $isProse = $quote === '' + && $match['separator'] === ':' + && ! in_array($match['prefix'][0], ['"', "'"], true) + && preg_match('/^[A-Za-z][a-z]*$/', $match['key']) === 1 + && ctype_alpha($value) + && $match['end'] === null; + + return $isProse ? $match[0] : $match['prefix'].$quote.REDACTED.$quote; + }, $text, flags: PREG_UNMATCHED_AS_NULL); + if ($result === null) { + throw new RuntimeException('Log redaction failed.'); + } + + return $result; + } + + private function redactPatterns(string $text): string + { + $patterns = [ + '/((?:Authorization|Proxy-Authorization)\s*:\s*)[^\r\n]+/i' => '$1'.REDACTED, + '/x-access-token:.*?(?=@)/' => 'x-access-token:'.REDACTED, + '/oauth2:.*?(?=@)/' => 'oauth2:'.REDACTED, + '/((?:https?|postgres|mysql|mongodb|rediss?|mariadb|ftp|sftp|ssh|amqp|amqps|ldap|ldaps|s3):\/\/[^:]+:)[^@]+(@)/i' => '$1'.REDACTED.'$2', + self::EMAIL_PATTERN => REDACTED, + '/Bearer\s+[A-Za-z0-9\-_]+\.[A-Za-z0-9\-_]+\.[A-Za-z0-9\-_]+/i' => 'Bearer '.REDACTED, + '/Bearer\s+[^\s,;]+/i' => 'Bearer '.REDACTED, + '/\bgh[pousr]_[A-Za-z0-9.\-_]{36,}(?![A-Za-z0-9.\-_])/' => REDACTED, + '/\b(gl(?:pat|cbt|rt)-[A-Za-z0-9\-_]{20,})\b/' => REDACTED, + '/\b(A(?:KIA|BIA|CCA|SIA)[A-Z0-9]{16})\b/' => REDACTED, + '/(aws_secret_access_key|AWS_SECRET_ACCESS_KEY)[=:]\s*[\'\"]?([A-Za-z0-9\/+\=]{40})[\'\"]?/i' => '$1='.REDACTED, + '/(api[_-]?key|apikey|api[_-]?secret|secret[_-]?key)[=:]\s*[\'\"]?[A-Za-z0-9\-_]{16,}[\'\"]?/i' => '$1='.REDACTED, + '/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/' => REDACTED, + ]; + if (! $this->redactEmails) { + unset($patterns[self::EMAIL_PATTERN]); + } + + foreach ($patterns as $pattern => $replacement) { + $text = preg_replace($pattern, $replacement, $text); + if ($text === null) { + throw new RuntimeException('Log redaction failed.'); + } + } + + return $text; + } + + private function isSensitiveKey(string $key): bool + { + $normalized = strtolower(preg_replace(['/([a-z0-9])([A-Z])/', '/[-\s]+/'], ['$1_$2', '_'], $key) ?? $key); + + foreach (self::SENSITIVE_KEYS as $sensitiveKey) { + if ($normalized === $sensitiveKey || str_ends_with($normalized, '_'.$sensitiveKey)) { + return true; + } + } + + return false; + } +} diff --git a/app/Services/ServerTransfer/ServerTransferMigrator.php b/app/Services/ServerTransfer/ServerTransferMigrator.php index 2ec3b6c63d..1c2571e3d0 100644 --- a/app/Services/ServerTransfer/ServerTransferMigrator.php +++ b/app/Services/ServerTransfer/ServerTransferMigrator.php @@ -4,6 +4,8 @@ namespace App\Services\ServerTransfer; use App\Models\Server; use App\Rules\SafeWebhookUrl; +use App\Services\Security\SensitiveDataRedactor; +use Illuminate\Database\QueryException; use Illuminate\Http\Client\ConnectionException; use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Validator; @@ -80,8 +82,10 @@ class ServerTransferMigrator targetInstanceUrl: $targetUrl, ); } catch (Throwable $e) { + // Query errors include SQL and bindings, so they are never shown to the user. + $reason = $e instanceof QueryException ? 'a database error occurred' : remove_iip($e->getMessage()); throw new RuntimeException( - "Server was imported on {$targetUrl}, but this instance could not mark it as transferred: {$e->getMessage()}. ". + "Server was imported on {$targetUrl}, but this instance could not mark it as transferred: {$reason}. ". 'Disable management of this server here (or API: POST /api/v1/servers/{uuid}/transfer/complete) so automations stay disabled. Do not re-import on the target.', previous: $e ); @@ -91,7 +95,7 @@ class ServerTransferMigrator 'server_uuid' => $server->uuid, 'export_id' => is_string($exportId) ? $exportId : null, 'target_url' => $targetUrl, - 'import' => $importBody, + 'import' => app(SensitiveDataRedactor::class)->redactValue($importBody), 'complete' => $complete, 'warnings' => $warnings, 'message' => 'Server migrated to '.$targetUrl.'. Automations disabled on this instance.', @@ -172,14 +176,14 @@ class ServerTransferMigrator 'adopt_mode' => $adoptMode, ]); } catch (ConnectionException $e) { - throw new RuntimeException("Could not reach target instance at {$importUrl}: {$e->getMessage()}", previous: $e); + throw new RuntimeException('Could not reach target instance at '.remove_iip($importUrl).': '.remove_iip($e->getMessage()), previous: $e); } catch (Throwable $e) { - throw new RuntimeException("Transfer to target failed: {$e->getMessage()}", previous: $e); + throw new RuntimeException('Transfer to target failed: '.remove_iip($e->getMessage()), previous: $e); } $importBody = $response->json(); if (! is_array($importBody)) { - $importBody = ['raw' => $response->body()]; + $importBody = ['raw' => remove_iip($response->body())]; } if (! $response->successful()) { @@ -189,7 +193,7 @@ class ServerTransferMigrator if (is_array($message)) { $message = json_encode($message); } - throw new RuntimeException("Target import failed (HTTP {$response->status()}): {$message}"); + throw new RuntimeException("Target import failed (HTTP {$response->status()}): ".remove_iip((string) $message)); } return $importBody; diff --git a/app/Support/ResourceStartActivity.php b/app/Support/ResourceStartActivity.php index 64b969587e..ac125bcc7f 100644 --- a/app/Support/ResourceStartActivity.php +++ b/app/Support/ResourceStartActivity.php @@ -158,12 +158,14 @@ class ResourceStartActivity } /** - * Mark an activity as failed, append the reason to its log and stop log polling. + * Mark an activity as failed, append the redacted reason to its log and stop log polling. * * @param array $extraProperties */ public static function markFailed(Activity $activity, string $message, array $extraProperties = []): void { + // Callers can pass exception or command output text, which can contain secrets. + $message = remove_iip($message); $properties = [ ...$extraProperties, 'status' => ProcessStatus::ERROR->value, diff --git a/app/Traits/ExecuteRemoteCommand.php b/app/Traits/ExecuteRemoteCommand.php index 449c9c55dd..86ed88e9b8 100644 --- a/app/Traits/ExecuteRemoteCommand.php +++ b/app/Traits/ExecuteRemoteCommand.php @@ -2,12 +2,13 @@ namespace App\Traits; +use App\Actions\CoolifyTask\RunRemoteProcess; use App\Enums\ApplicationDeploymentStatus; use App\Exceptions\DeploymentException; use App\Helpers\SshMultiplexingHelper; use App\Models\EnvironmentVariable; use App\Models\Server; -use Carbon\Carbon; +use App\Services\Security\SensitiveDataRedactor; use Illuminate\Support\Collection; use Illuminate\Support\Facades\Process; use Illuminate\Support\Stringable; @@ -22,52 +23,44 @@ trait ExecuteRemoteCommand public static int $batch_counter = 0; - private function redact_sensitive_info($text) + /** @var array{known: array, locked: array}|null */ + private ?array $logRedactionSecrets = null; + + /** @var array|null */ + private ?array $remoteSecretLogRedactionValues = null; + + private function redact_sensitive_info(string $text): string { try { - $text = remove_iip($text); - - if (! isset($this->application)) { - return $text; - } - - $lockedVars = collect([]); - - if (isset($this->application->environment_variables)) { - $lockedVars = $lockedVars->merge( - $this->application->environment_variables - ->where('is_shown_once', true) - ->flatMap(fn (EnvironmentVariable $variable): array => $variable->logRedactionValues()) - ->filter() - ); - } - - if (isset($this->pull_request_id) && $this->pull_request_id !== 0 && isset($this->application->environment_variables_preview)) { - $lockedVars = $lockedVars->merge( - $this->application->environment_variables_preview - ->where('is_shown_once', true) - ->flatMap(fn (EnvironmentVariable $variable): array => $variable->logRedactionValues()) - ->filter() - ); - } - - if (isset($this->remote_secrets_cache)) { - $lockedVars = $lockedVars->merge(EnvironmentVariable::remoteSecretLogRedactionValues($this->remote_secrets_cache)); - } - - foreach ($lockedVars as $key => $value) { - $escapedValue = preg_quote($value, '/'); - $text = preg_replace( - '/'.$escapedValue.'/', - REDACTED, - $text - ); - } - - return is_string($text) ? $text : REDACTED; + // Remote secrets come from the job's remote_secrets_cache below, so they are not fetched again. + $this->logRedactionSecrets ??= EnvironmentVariable::logRedactionSecretsFor( + $this->application ?? null, + isset($this->pull_request_id) && $this->pull_request_id !== 0, + resolveRemoteSecrets: false, + ); } catch (\Throwable) { return REDACTED; } + + $remoteSecrets = isset($this->remote_secrets_cache) + ? ($this->remoteSecretLogRedactionValues ??= EnvironmentVariable::remoteSecretLogRedactionValues($this->remote_secrets_cache)) + : []; + + return app(SensitiveDataRedactor::class)->redactText( + $text, + $this->logRedactionSecrets['known'], + array_merge($this->logRedactionSecrets['locked'], $remoteSecrets), + ); + } + + /** + * Remote secrets are never stored in Coolify, so the deployment log model must get them from the job. + */ + private function registerRemoteSecretsForLogs(): void + { + if (isset($this->application_deployment_queue, $this->remote_secrets_cache)) { + $this->application_deployment_queue->redactRemoteSecrets($this->remote_secrets_cache); + } } public function execute_remote_command(...$commands) @@ -81,9 +74,7 @@ trait ExecuteRemoteCommand if ($this->server instanceof Server === false) { throw new \RuntimeException('Server is not set or is not an instance of Server model'); } - if (isset($this->application_deployment_queue, $this->remote_secrets_cache)) { - $this->application_deployment_queue->redactRemoteSecrets($this->remote_secrets_cache); - } + $this->registerRemoteSecretsForLogs(); $commandsText->each(function ($single_command) { $command = data_get($single_command, 'command') ?? $single_command[0] ?? null; if ($command === null) { @@ -181,57 +172,46 @@ trait ExecuteRemoteCommand $remote_command = $input === null ? SshMultiplexingHelper::generateSshCommand($this->server, $command) : SshMultiplexingHelper::generateSshStdinCommand($this->server, $command); - $process = Process::timeout(config('constants.ssh.command_timeout'))->idleTimeout(3600)->input($input)->start($remote_command, function (string $type, string $output) use ($command, $hidden, $customType, $append, $command_hidden, $skip_command_log) { + $addLogEntry = function (string $type, string $output) use ($command, $hidden, $customType, $command_hidden, $skip_command_log): void { // Sanitize output to ensure valid UTF-8 encoding before JSON encoding - $sanitized_output = sanitize_utf8_text($output); - $log_output = str($sanitized_output)->trim(); + $log_output = str(sanitize_utf8_text($output))->trim(); if ($log_output->startsWith('╔')) { $log_output = "\n".$log_output; } - $new_log_entry = [ - 'command' => $skip_command_log || $command_hidden ? null : $this->redact_sensitive_info($command), - 'output' => $this->redact_sensitive_info($skip_command_log ? $this->redactSensitiveCommandPayloads((string) $log_output, (string) $command) : $log_output), - 'type' => $customType ?? ($type === 'err' ? 'stderr' : 'stdout'), - 'timestamp' => Carbon::now('UTC'), - 'hidden' => $hidden, - 'batch' => static::$batch_counter, - ]; - if (! $this->application_deployment_queue->logs) { - $new_log_entry['order'] = 1; - } else { - try { - $previous_logs = json_decode($this->application_deployment_queue->logs, associative: true, flags: JSON_THROW_ON_ERROR); - } catch (\JsonException $e) { - // If existing logs are corrupted, start fresh - $previous_logs = []; - $new_log_entry['order'] = 1; - } - if (is_array($previous_logs)) { - $new_log_entry['order'] = count($previous_logs) + 1; - } else { - $previous_logs = []; - $new_log_entry['order'] = 1; - } - } - $previous_logs[] = $new_log_entry; - - try { - $this->application_deployment_queue->logs = json_encode($previous_logs, flags: JSON_THROW_ON_ERROR); - } catch (\JsonException $e) { - // If JSON encoding still fails, use fallback with invalid sequences replacement - $this->application_deployment_queue->logs = json_encode($previous_logs, flags: JSON_INVALID_UTF8_SUBSTITUTE); + $this->application_deployment_queue->addLogEntry( + message: $skip_command_log ? $this->redactSensitiveCommandPayloads((string) $log_output, (string) $command) : (string) $log_output, + type: $customType ?? ($type === 'err' ? 'stderr' : 'stdout'), + hidden: $hidden, + command: $skip_command_log || $command_hidden ? null : $command, + batch: static::$batch_counter, + ); + }; + // Output is logged per complete line, so a secret split across two output chunks is still redacted. + $pending = []; + $process = Process::timeout(config('constants.ssh.command_timeout'))->idleTimeout(3600)->input($input)->start($remote_command, function (string $type, string $output) use ($addLogEntry, $append, &$pending) { + $pending[$type] = ($pending[$type] ?? '').$output; + $complete_output = RunRemoteProcess::takeCompleteOutput($pending[$type]); + if ($complete_output !== '') { + $addLogEntry($type, $complete_output); } - $this->application_deployment_queue->save(); - - $this->saveCommandOutput($sanitized_output, $append); + $this->saveCommandOutput(sanitize_utf8_text($output), $append); }); $this->application_deployment_queue->update([ 'current_process_id' => $process->id(), ]); - $process_result = $process->wait(); + try { + $process_result = $process->wait(); + } finally { + foreach ($pending as $type => $output) { + $remaining_output = RunRemoteProcess::takeCompleteOutput($pending[$type], final: true); + if ($remaining_output !== '') { + $addLogEntry($type, $remaining_output); + } + } + } if ($process_result->exitCode() !== 0) { if (! $ignore_errors) { // Check if deployment was cancelled while command was running @@ -324,40 +304,10 @@ trait ExecuteRemoteCommand { $retryMessage = "SSH connection failed. Retrying... (Attempt {$attempt}/{$maxRetries}, waiting {$delay}s)\nError: {$errorMessage}"; - $new_log_entry = [ - 'output' => $this->redact_sensitive_info($retryMessage), - 'type' => 'stdout', - 'timestamp' => Carbon::now('UTC'), - 'hidden' => false, - 'batch' => static::$batch_counter, - ]; - - if (! $this->application_deployment_queue->logs) { - $new_log_entry['order'] = 1; - $previous_logs = []; - } else { - try { - $previous_logs = json_decode($this->application_deployment_queue->logs, associative: true, flags: JSON_THROW_ON_ERROR); - } catch (\JsonException $e) { - $previous_logs = []; - $new_log_entry['order'] = 1; - } - if (is_array($previous_logs)) { - $new_log_entry['order'] = count($previous_logs) + 1; - } else { - $previous_logs = []; - $new_log_entry['order'] = 1; - } - } - - $previous_logs[] = $new_log_entry; - - try { - $this->application_deployment_queue->logs = json_encode($previous_logs, flags: JSON_THROW_ON_ERROR); - } catch (\JsonException $e) { - $this->application_deployment_queue->logs = json_encode($previous_logs, flags: JSON_INVALID_UTF8_SUBSTITUTE); - } - - $this->application_deployment_queue->save(); + $this->registerRemoteSecretsForLogs(); + $this->application_deployment_queue->addLogEntry( + message: $retryMessage, + batch: static::$batch_counter, + ); } } diff --git a/app/Traits/HandlesNotificationResponse.php b/app/Traits/HandlesNotificationResponse.php index 77d8b8477d..8cbc7e2968 100644 --- a/app/Traits/HandlesNotificationResponse.php +++ b/app/Traits/HandlesNotificationResponse.php @@ -2,10 +2,15 @@ namespace App\Traits; +use App\Rules\SafeWebhookUrl; +use Closure; +use GuzzleHttp\Exception\TransferException; +use Illuminate\Http\Client\ConnectionException; +use Illuminate\Http\Client\RequestException; use Illuminate\Http\Client\Response; use Illuminate\Support\Facades\Log; -use Illuminate\Support\Str; use RuntimeException; +use Symfony\Component\HttpFoundation\Response as HttpResponse; /** * Shared response handling for the notification jobs. The job must have a @@ -14,6 +19,28 @@ use RuntimeException; */ trait HandlesNotificationResponse { + /** + * Send the request and replace HTTP client exceptions, whose messages contain + * the full URL (bot token, secret webhook path), with one that only names the + * scheme and host. The original exception is not kept as `previous`, so the + * URL never reaches failed_jobs or Sentry. + * + * @param Closure(): Response $send + */ + protected function sendNotificationRequest(string $channel, string $url, Closure $send): Response + { + try { + return $send(); + } catch (ConnectionException|RequestException|TransferException $e) { + throw new RuntimeException(sprintf( + '%s notification request to %s failed (%s).', + $channel, + SafeWebhookUrl::redactedUrlForLog($url), + class_basename($e), + )); + } + } + /** * Rate limits and server errors are retried by the queue. A configuration * error (invalid token, deleted webhook) counts as a delivery failure and @@ -32,7 +59,8 @@ trait HandlesNotificationResponse throw new RuntimeException("{$channel} notification failed with {$response->status()} status code."); } - $reason = "{$channel} returned HTTP {$response->status()}: ".Str::limit($response->body(), 200); + // The response body is never part of the reason: it can echo tokens or webhook secrets. + $reason = "{$channel} returned HTTP {$response->status()} (".(HttpResponse::$statusTexts[$response->status()] ?? 'Client Error').').'; if ($isConfigurationError) { $this->settings?->recordDeliveryFailure($reason); diff --git a/app/Traits/HasLogRedactionSetting.php b/app/Traits/HasLogRedactionSetting.php new file mode 100644 index 0000000000..5077bb0922 --- /dev/null +++ b/app/Traits/HasLogRedactionSetting.php @@ -0,0 +1,23 @@ +mergeFillable(['redact_all_env_values_in_logs']); + $this->mergeCasts(['redact_all_env_values_in_logs' => 'boolean']); + } + + /** + * Tell if every environment value is hidden in logs. When false, only locked values are hidden. + */ + public function redactsAllEnvValuesInLogs(): bool + { + return $this->redact_all_env_values_in_logs ?? true; + } +} diff --git a/app/Traits/TracksDeliveryFailures.php b/app/Traits/TracksDeliveryFailures.php index 5b8c785437..8ab56705d4 100644 --- a/app/Traits/TracksDeliveryFailures.php +++ b/app/Traits/TracksDeliveryFailures.php @@ -40,8 +40,13 @@ trait TracksDeliveryFailures }); } + /** + * The reason must not contain a provider response body. It is shared with the + * team, the team admins, and the instance admins, so it is redacted once more. + */ public function recordDeliveryFailure(string $reason): void { + $reason = remove_iip($reason); $this->increment('failure_count'); Log::warning('Notification delivery failed', [ diff --git a/bootstrap/helpers/api.php b/bootstrap/helpers/api.php index 388891c208..ca356fa16a 100644 --- a/bootstrap/helpers/api.php +++ b/bootstrap/helpers/api.php @@ -203,6 +203,7 @@ function sharedDataApplications(?Application $application = null): array 'inject_build_args_to_dockerfile' => 'boolean', 'include_source_commit_in_build' => 'boolean', 'is_env_sorting_enabled' => 'boolean', + 'redact_all_env_values_in_logs' => 'boolean', 'is_pr_deployments_public_enabled' => 'boolean', 'is_gzip_enabled' => 'boolean', 'is_stripprefix_enabled' => 'boolean', @@ -478,6 +479,7 @@ function removeUnnecessaryFieldsFromRequest(Request $request) $request->offsetUnset('disable_build_cache'); $request->offsetUnset('inject_build_args_to_dockerfile'); $request->offsetUnset('is_env_sorting_enabled'); + $request->offsetUnset('redact_all_env_values_in_logs'); $request->offsetUnset('is_pr_deployments_public_enabled'); $request->offsetUnset('stop_grace_period'); $request->offsetUnset('docker_images_to_keep'); diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.php index d17c4188ac..00652f46b5 100644 --- a/bootstrap/helpers/remoteProcess.php +++ b/bootstrap/helpers/remoteProcess.php @@ -9,6 +9,7 @@ use App\Models\Application; use App\Models\ApplicationDeploymentQueue; use App\Models\PrivateKey; use App\Models\Server; +use App\Services\Security\SensitiveDataRedactor; use App\Support\RemoteProcessCommand; use Carbon\Carbon; use Illuminate\Contracts\Process\ProcessResult; @@ -371,45 +372,7 @@ function decode_remote_command_output(?ApplicationDeploymentQueue $application_d function remove_iip($text) { - // Ensure the input is valid UTF-8 before processing - $text = sanitize_utf8_text($text); - - // Git access tokens - $text = preg_replace('/x-access-token:.*?(?=@)/', 'x-access-token:'.REDACTED, $text); - $text = preg_replace('/oauth2:.*?(?=@)/', 'oauth2:'.REDACTED, $text); - - // ANSI color codes - $text = preg_replace('/\x1b\[[0-9;]*m/', '', $text); - - // Generic URLs with passwords (covers database URLs, ftp, amqp, ssh, git basic auth, etc.) - // (protocol://user:password@host → protocol://user:@host) - $text = preg_replace('/((?:https?|postgres|mysql|mongodb|rediss?|mariadb|ftp|sftp|ssh|amqp|amqps|ldap|ldaps|s3):\/\/[^:]+:)[^@]+(@)/i', '$1'.REDACTED.'$2', $text); - - // Email addresses - $text = preg_replace('/[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/', REDACTED, $text); - - // Bearer/JWT tokens - $text = preg_replace('/Bearer\s+[A-Za-z0-9\-_]+\.[A-Za-z0-9\-_]+\.[A-Za-z0-9\-_]+/i', 'Bearer '.REDACTED, $text); - - // GitHub tokens (ghp_ = personal, gho_ = OAuth, ghu_ = user-to-server, ghs_ = server-to-server, ghr_ = refresh) - $text = preg_replace('/\bgh[pousr]_[A-Za-z0-9.\-_]{36,}(?![A-Za-z0-9.\-_])/', REDACTED, $text); - - // GitLab tokens (glpat- = personal access token, glcbt- = CI build token, glrt- = runner token) - $text = preg_replace('/\b(gl(?:pat|cbt|rt)-[A-Za-z0-9\-_]{20,})\b/', REDACTED, $text); - - // AWS credentials (Access Key ID starts with AKIA, ABIA, ACCA, ASIA) - $text = preg_replace('/\b(A(?:KIA|BIA|CCA|SIA)[A-Z0-9]{16})\b/', REDACTED, $text); - - // AWS Secret Access Key (40 character base64-ish string, typically follows access key) - $text = preg_replace('/(aws_secret_access_key|AWS_SECRET_ACCESS_KEY)[=:]\s*[\'"]?([A-Za-z0-9\/+=]{40})[\'"]?/i', '$1='.REDACTED, $text); - - // API keys (common patterns) - $text = preg_replace('/(api[_-]?key|apikey|api[_-]?secret|secret[_-]?key)[=:]\s*[\'"]?[A-Za-z0-9\-_]{16,}[\'"]?/i', '$1='.REDACTED, $text); - - // Private key blocks - $text = preg_replace('/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/', REDACTED, $text); - - return $text; + return app(SensitiveDataRedactor::class)->redactText((string) $text); } /** diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index 0b2108bfb9..69d653d668 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -38,6 +38,7 @@ use App\Models\StandaloneSqlite; use App\Models\SwarmDocker; use App\Models\Team; use App\Models\User; +use App\Services\Security\SensitiveDataRedactor; use App\Support\DnsRecordHints; use Carbon\CarbonImmutable; use DanHarrin\LivewireRateLimiting\Exceptions\TooManyRequestsException; @@ -1716,10 +1717,14 @@ function removeAnsiColors($text) return preg_replace('/\e[[][A-Za-z0-9];?[0-9]*m?/', '', $text); } -function sanitizeLogsForExport(string $text): string +/** + * Redact log text before it leaves Coolify (copy, download, API, MCP). + * + * @param array{known?: array, locked?: array} $secrets Resource values to hide, usually from EnvironmentVariable::logRedactionSecretsFor(). + */ +function sanitizeLogsForExport(string $text, array $secrets = []): string { - // All sanitization is now handled by remove_iip() - return remove_iip($text); + return app(SensitiveDataRedactor::class)->redactText($text, $secrets['known'] ?? [], $secrets['locked'] ?? []); } function getTopLevelNetworks(Service|Application $resource): Collection diff --git a/config/logging.php b/config/logging.php index 40d372b58d..497d4bbdd5 100644 --- a/config/logging.php +++ b/config/logging.php @@ -1,5 +1,7 @@ (bool) env('LOG_REDACTION_ENABLED', true), + 'channels' => [ 'stack' => [ 'driver' => 'stack', @@ -61,6 +76,7 @@ return [ 'driver' => 'single', 'path' => storage_path('logs/laravel.log'), 'level' => env('LOG_LEVEL', 'debug'), + 'tap' => [RedactSensitiveData::class], ], 'daily' => [ @@ -68,6 +84,7 @@ return [ 'path' => storage_path('logs/laravel.log'), 'level' => env('LOG_LEVEL', 'debug'), 'days' => 14, + 'tap' => [RedactSensitiveData::class], ], 'slack' => [ @@ -76,11 +93,13 @@ return [ 'username' => 'Laravel Log', 'emoji' => ':boom:', 'level' => env('LOG_LEVEL', 'critical'), + 'tap' => [RedactSensitiveData::class], ], 'papertrail' => [ 'driver' => 'monolog', 'level' => env('LOG_LEVEL', 'debug'), + 'tap' => [RedactSensitiveData::class], 'handler' => env('LOG_PAPERTRAIL_HANDLER', SyslogUdpHandler::class), 'handler_with' => [ 'host' => env('PAPERTRAIL_URL'), @@ -92,6 +111,7 @@ return [ 'stderr' => [ 'driver' => 'monolog', 'level' => env('LOG_LEVEL', 'debug'), + 'tap' => [RedactSensitiveData::class], 'handler' => StreamHandler::class, 'formatter' => env('LOG_STDERR_FORMATTER'), 'with' => [ @@ -102,12 +122,14 @@ return [ 'syslog' => [ 'driver' => 'syslog', 'level' => env('LOG_LEVEL', 'debug'), + 'tap' => [RedactSensitiveData::class], 'facility' => LOG_USER, ], 'errorlog' => [ 'driver' => 'errorlog', 'level' => env('LOG_LEVEL', 'debug'), + 'tap' => [RedactSensitiveData::class], ], 'null' => [ @@ -124,6 +146,7 @@ return [ 'path' => storage_path('logs/scheduled.log'), 'level' => 'debug', 'days' => 7, + 'tap' => [RedactSensitiveData::class], ], 'scheduled-errors' => [ @@ -131,6 +154,7 @@ return [ 'path' => storage_path('logs/scheduled-errors.log'), 'level' => 'warning', 'days' => 14, + 'tap' => [RedactSensitiveData::class], ], 'audit' => [ @@ -139,6 +163,7 @@ return [ 'level' => env('LOG_AUDIT_LEVEL', 'info'), 'days' => env('LOG_AUDIT_DAYS', 90), 'replace_placeholders' => true, + 'tap' => [RedactSensitiveAuditData::class], ], ], diff --git a/database/migrations/2026_09_25_105051_add_redact_all_env_values_in_logs_to_resources.php b/database/migrations/2026_09_25_105051_add_redact_all_env_values_in_logs_to_resources.php new file mode 100644 index 0000000000..0944bc2daa --- /dev/null +++ b/database/migrations/2026_09_25_105051_add_redact_all_env_values_in_logs_to_resources.php @@ -0,0 +1,34 @@ +boolean('redact_all_env_values_in_logs')->default(true); + }); + Schema::table('services', function (Blueprint $table) { + $table->boolean('redact_all_env_values_in_logs')->default(true); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('application_settings', function (Blueprint $table) { + $table->dropColumn('redact_all_env_values_in_logs'); + }); + Schema::table('services', function (Blueprint $table) { + $table->dropColumn('redact_all_env_values_in_logs'); + }); + } +}; diff --git a/database/migrations/2026_10_12_154805_add_application_id_index_to_application_settings_table.php b/database/migrations/2026_10_12_154805_add_application_id_index_to_application_settings_table.php new file mode 100644 index 0000000000..da19a54fb4 --- /dev/null +++ b/database/migrations/2026_10_12_154805_add_application_id_index_to_application_settings_table.php @@ -0,0 +1,36 @@ +index('application_id'); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + if (! Schema::hasIndex('application_settings', ['application_id'])) { + return; + } + + Schema::table('application_settings', function (Blueprint $table) { + $table->dropIndex(['application_id']); + }); + } +}; diff --git a/database/migrations/2026_10_13_101731_add_redact_all_env_values_in_logs_to_standalone_databases.php b/database/migrations/2026_10_13_101731_add_redact_all_env_values_in_logs_to_standalone_databases.php new file mode 100644 index 0000000000..28af764121 --- /dev/null +++ b/database/migrations/2026_10_13_101731_add_redact_all_env_values_in_logs_to_standalone_databases.php @@ -0,0 +1,50 @@ +tables as $table) { + if (Schema::hasColumn($table, 'redact_all_env_values_in_logs')) { + continue; + } + Schema::table($table, function (Blueprint $table) { + $table->boolean('redact_all_env_values_in_logs')->default(true); + }); + } + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + foreach ($this->tables as $table) { + if (! Schema::hasColumn($table, 'redact_all_env_values_in_logs')) { + continue; + } + Schema::table($table, function (Blueprint $table) { + $table->dropColumn('redact_all_env_values_in_logs'); + }); + } + } +}; diff --git a/openapi.json b/openapi.json index dabc004193..2521a0fdb7 100644 --- a/openapi.json +++ b/openapi.json @@ -886,6 +886,10 @@ "type": "boolean", "description": "Sort environment variables." }, + "redact_all_env_values_in_logs": { + "type": "boolean", + "description": "Redact all environment variable values in logs. When false, only locked (shown once) values are redacted. Requires team admin permissions." + }, "is_pr_deployments_public_enabled": { "type": "boolean", "description": "Make pull request deployments public." @@ -1512,6 +1516,10 @@ "type": "boolean", "description": "Sort environment variables." }, + "redact_all_env_values_in_logs": { + "type": "boolean", + "description": "Redact all environment variable values in logs. When false, only locked (shown once) values are redacted. Requires team admin permissions." + }, "is_pr_deployments_public_enabled": { "type": "boolean", "description": "Make pull request deployments public." @@ -2138,6 +2146,10 @@ "type": "boolean", "description": "Sort environment variables." }, + "redact_all_env_values_in_logs": { + "type": "boolean", + "description": "Redact all environment variable values in logs. When false, only locked (shown once) values are redacted. Requires team admin permissions." + }, "is_pr_deployments_public_enabled": { "type": "boolean", "description": "Make pull request deployments public." @@ -2659,6 +2671,10 @@ "type": "boolean", "description": "Sort environment variables." }, + "redact_all_env_values_in_logs": { + "type": "boolean", + "description": "Redact all environment variable values in logs. When false, only locked (shown once) values are redacted. Requires team admin permissions." + }, "is_pr_deployments_public_enabled": { "type": "boolean", "description": "Make pull request deployments public." @@ -3160,6 +3176,10 @@ "type": "boolean", "description": "Sort environment variables." }, + "redact_all_env_values_in_logs": { + "type": "boolean", + "description": "Redact all environment variable values in logs. When false, only locked (shown once) values are redacted. Requires team admin permissions." + }, "is_pr_deployments_public_enabled": { "type": "boolean", "description": "Make pull request deployments public." @@ -3932,6 +3952,10 @@ "type": "boolean", "description": "Sort environment variables." }, + "redact_all_env_values_in_logs": { + "type": "boolean", + "description": "Redact all environment variable values in logs. When false, only locked (shown once) values are redacted. Requires team admin permissions." + }, "is_pr_deployments_public_enabled": { "type": "boolean", "description": "Make pull request deployments public." @@ -7418,6 +7442,10 @@ "description": "Healthcheck start period in seconds.", "minimum": 0, "default": 5 + }, + "redact_all_env_values_in_logs": { + "type": "boolean", + "description": "Redact all environment variable values in logs. When false, only locked (shown once) values, database passwords, and remote secrets are redacted. Requires team admin permissions." } }, "type": "object" @@ -9112,7 +9140,7 @@ "Databases" ], "summary": "List backup executions", - "description": "Get all executions for a specific backup configuration.", + "description": "Get all executions for a specific backup configuration. The execution `message` is only returned (redacted) to tokens with the `read:sensitive` or `root` ability; otherwise it is `null`.", "operationId": "list-backup-executions", "parameters": [ { @@ -9158,7 +9186,11 @@ "type": "string" }, "message": { - "type": "string" + "description": "Redacted execution output. Null unless the token has the `read:sensitive` or `root` ability.", + "type": [ + "string", + "null" + ] }, "status": { "type": "string" @@ -19657,6 +19689,11 @@ "default": true, "description": "Escape special characters in labels. By default, $ (and other chars) is escaped. If you want to use env variables inside the labels, turn this off." }, + "redact_all_env_values_in_logs": { + "type": "boolean", + "default": true, + "description": "Redact all environment variable values in logs. When false, only locked (shown once) values are redacted. Requires team admin permissions." + }, "tags": { "type": "array", "items": { @@ -19984,6 +20021,11 @@ "type": "boolean", "default": true, "description": "Escape special characters in labels. By default, $ (and other chars) is escaped. If you want to use env variables inside the labels, turn this off." + }, + "redact_all_env_values_in_logs": { + "type": "boolean", + "default": true, + "description": "Redact all environment variable values in logs. When false, only locked (shown once) values are redacted. Requires team admin permissions." } }, "type": "object" @@ -24439,6 +24481,10 @@ "use_build_secrets": { "type": "boolean" }, + "redact_all_env_values_in_logs": { + "type": "boolean", + "description": "Hide every environment variable value in logs. When false, only locked values are hidden." + }, "inject_build_args_to_dockerfile": { "type": "boolean" }, @@ -25024,6 +25070,10 @@ "type": "boolean", "description": "The flag to enable the container label escape." }, + "redact_all_env_values_in_logs": { + "type": "boolean", + "description": "Hide every environment variable value in logs. When false, only locked values are hidden." + }, "is_container_label_readonly_enabled": { "type": "boolean", "description": "The flag to enable the container label readonly." diff --git a/openapi.yaml b/openapi.yaml index 28a3ade16b..598df0a082 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -572,6 +572,9 @@ paths: is_env_sorting_enabled: type: boolean description: 'Sort environment variables.' + redact_all_env_values_in_logs: + type: boolean + description: 'Redact all environment variable values in logs. When false, only locked (shown once) values are redacted. Requires team admin permissions.' is_pr_deployments_public_enabled: type: boolean description: 'Make pull request deployments public.' @@ -985,6 +988,9 @@ paths: is_env_sorting_enabled: type: boolean description: 'Sort environment variables.' + redact_all_env_values_in_logs: + type: boolean + description: 'Redact all environment variable values in logs. When false, only locked (shown once) values are redacted. Requires team admin permissions.' is_pr_deployments_public_enabled: type: boolean description: 'Make pull request deployments public.' @@ -1398,6 +1404,9 @@ paths: is_env_sorting_enabled: type: boolean description: 'Sort environment variables.' + redact_all_env_values_in_logs: + type: boolean + description: 'Redact all environment variable values in logs. When false, only locked (shown once) values are redacted. Requires team admin permissions.' is_pr_deployments_public_enabled: type: boolean description: 'Make pull request deployments public.' @@ -1752,6 +1761,9 @@ paths: is_env_sorting_enabled: type: boolean description: 'Sort environment variables.' + redact_all_env_values_in_logs: + type: boolean + description: 'Redact all environment variable values in logs. When false, only locked (shown once) values are redacted. Requires team admin permissions.' is_pr_deployments_public_enabled: type: boolean description: 'Make pull request deployments public.' @@ -2092,6 +2104,9 @@ paths: is_env_sorting_enabled: type: boolean description: 'Sort environment variables.' + redact_all_env_values_in_logs: + type: boolean + description: 'Redact all environment variable values in logs. When false, only locked (shown once) values are redacted. Requires team admin permissions.' is_pr_deployments_public_enabled: type: boolean description: 'Make pull request deployments public.' @@ -2605,6 +2620,9 @@ paths: is_env_sorting_enabled: type: boolean description: 'Sort environment variables.' + redact_all_env_values_in_logs: + type: boolean + description: 'Redact all environment variable values in logs. When false, only locked (shown once) values are redacted. Requires team admin permissions.' is_pr_deployments_public_enabled: type: boolean description: 'Make pull request deployments public.' @@ -4805,6 +4823,9 @@ paths: description: 'Healthcheck start period in seconds.' minimum: 0 default: 5 + redact_all_env_values_in_logs: + type: boolean + description: 'Redact all environment variable values in logs. When false, only locked (shown once) values, database passwords, and remote secrets are redacted. Requires team admin permissions.' type: object responses: '200': @@ -5992,7 +6013,7 @@ paths: tags: - Databases summary: 'List backup executions' - description: 'Get all executions for a specific backup configuration.' + description: 'Get all executions for a specific backup configuration. The execution `message` is only returned (redacted) to tokens with the `read:sensitive` or `root` ability; otherwise it is `null`.' operationId: list-backup-executions parameters: - @@ -6016,7 +6037,7 @@ paths: application/json: schema: properties: - executions: { type: array, items: { properties: { uuid: { type: string }, filename: { type: string }, size: { type: integer }, created_at: { type: string }, message: { type: string }, status: { type: string } }, type: object } } + executions: { type: array, items: { properties: { uuid: { type: string }, filename: { type: string }, size: { type: integer }, created_at: { type: string }, message: { description: 'Redacted execution output. Null unless the token has the `read:sensitive` or `root` ability.', type: [string, 'null'] }, status: { type: string } }, type: object } } type: object '404': description: 'Backup configuration not found.' @@ -12438,6 +12459,10 @@ paths: type: boolean default: true description: 'Escape special characters in labels. By default, $ (and other chars) is escaped. If you want to use env variables inside the labels, turn this off.' + redact_all_env_values_in_logs: + type: boolean + default: true + description: 'Redact all environment variable values in logs. When false, only locked (shown once) values are redacted. Requires team admin permissions.' tags: type: array items: { type: string } @@ -12624,6 +12649,10 @@ paths: type: boolean default: true description: 'Escape special characters in labels. By default, $ (and other chars) is escaped. If you want to use env variables inside the labels, turn this off.' + redact_all_env_values_in_logs: + type: boolean + default: true + description: 'Redact all environment variable values in logs. When false, only locked (shown once) values are redacted. Requires team admin permissions.' type: object responses: '200': @@ -15609,6 +15638,9 @@ components: type: boolean use_build_secrets: type: boolean + redact_all_env_values_in_logs: + type: boolean + description: 'Hide every environment variable value in logs. When false, only locked values are hidden.' inject_build_args_to_dockerfile: type: boolean include_source_commit_in_build: @@ -16028,6 +16060,9 @@ components: is_container_label_escape_enabled: type: boolean description: 'The flag to enable the container label escape.' + redact_all_env_values_in_logs: + type: boolean + description: 'Hide every environment variable value in logs. When false, only locked values are hidden.' is_container_label_readonly_enabled: type: boolean description: 'The flag to enable the container label readonly.' diff --git a/resources/js/runtime-logs.js b/resources/js/runtime-logs.js index 0dad173ada..3b34fce197 100644 --- a/resources/js/runtime-logs.js +++ b/resources/js/runtime-logs.js @@ -348,9 +348,9 @@ export function initializeRuntimeLogsComponent() { }); }, - visibleText() { + visibleText(source = filtered) { const withTime = this.$wire.showTimeStamps; - return filtered + return source .map((line) => (withTime && line.ts ? `${formatTimestamp(line.ts)} ${line.text}` : line.text)) .join('\n'); }, @@ -406,9 +406,21 @@ export function initializeRuntimeLogsComponent() { }); }, - downloadLogs() { + // The browser holds unredacted lines, so fetch the redacted copy from the server + // and apply the same level and search filters before downloading it. + async downloadLogs() { + let logs; + try { + logs = await this.$wire.copyLogs(); + } catch { + window.Livewire.dispatch('error', ['Failed to prepare logs for download.']); + return; + } + if (!logs) return; + let id = 0; + const exported = filterLogLines(parseLogLines(logs, () => ++id), this.logFilters, this.searchQuery); const timestamp = new Date().toISOString().slice(0, 19).replace(/[T:]/g, '-'); - downloadText(this.visibleText() + '\n', `${this.containerName}-logs-${timestamp}.txt`); + downloadText(this.visibleText(exported) + '\n', `${this.containerName}-logs-${timestamp}.txt`); }, makeFullscreen() { diff --git a/resources/views/livewire/project/shared/environment-variable/all.blade.php b/resources/views/livewire/project/shared/environment-variable/all.blade.php index a9f89761d9..4d0b12d3be 100644 --- a/resources/views/livewire/project/shared/environment-variable/all.blade.php +++ b/resources/views/livewire/project/shared/environment-variable/all.blade.php @@ -28,27 +28,40 @@ @endcan @if ($view === 'normal') - @if ($resourceClass === 'App\Models\Application') -
- @if (data_get($resource, 'build_pack') !== 'dockercompose') - + @if ($resourceClass === 'App\Models\Application') +
+ @if (data_get($resource, 'build_pack') !== 'dockercompose') + + @endif + - @endif - -
- @else -

Manage this resource's environment variables below.

- @endif +
+ @else +

Manage this resource's environment variables below.

+ @endif + @if ($this->hasLogRedactionSetting()) +
+ +
+ @endif + @else
@can('manageEnvironment', $resource) diff --git a/resources/views/livewire/project/shared/get-logs.blade.php b/resources/views/livewire/project/shared/get-logs.blade.php index 3f855abea3..3249e8cf4b 100644 --- a/resources/views/livewire/project/shared/get-logs.blade.php +++ b/resources/views/livewire/project/shared/get-logs.blade.php @@ -194,6 +194,7 @@
+

Copied and downloaded logs hide secrets. The live view shows the raw output.