fix(deployments): defer container cleanup and configure dashboard HTTPS

This commit is contained in:
Andras Bacsai
2026-08-18 12:51:54 +02:00
parent 264cb8e2b0
commit d2d6c8ac96
10 changed files with 341 additions and 10 deletions
+35 -5
View File
@@ -52,6 +52,8 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
private const RAILPACK_GENERATED_CONFIG_PATH = '.coolify/railpack.generated.json';
private const CONTAINER_REMOVE_TIMEOUT_MARKER = '__COOLIFY_CONTAINER_REMOVE_TIMEOUT__';
private const DOCKER_CLIENT_ENV_KEYS = [
'BUILDKIT_HOST',
'BUILDX_BUILDER',
@@ -3977,15 +3979,45 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
);
} else {
$this->execute_remote_command(
[dockerStopCommand($timeout, $containerName, $this->server), 'hidden' => true, 'ignore_errors' => true],
["docker rm -f $containerName", 'hidden' => true, 'ignore_errors' => true]
[dockerStopCommand($timeout, $containerName, $this->server), 'hidden' => true, 'ignore_errors' => true]
);
$this->removeContainerWithTimeout($containerName);
}
} catch (Exception $error) {
$this->application_deployment_queue->addLogEntry("Error stopping container $containerName: ".$error->getMessage(), 'stderr');
}
}
private function removeContainerWithTimeout(string $containerName): void
{
$outputKey = 'container_remove_'.md5($containerName);
$this->execute_remote_command([
dockerRemoveCommandWithTimeout($containerName),
'hidden' => true,
'ignore_errors' => true,
'save' => $outputKey,
'append' => false,
]);
if (! isset($this->saved_outputs)) {
return;
}
$output = (string) $this->saved_outputs->get($outputKey, '');
if (! str_contains($output, self::CONTAINER_REMOVE_TIMEOUT_MARKER)) {
return;
}
$this->application_deployment_queue->addLogEntry(
"Warning: Removing container {$containerName} timed out after 60 seconds. The deployment will continue and cleanup will be retried in 5 minutes.",
'stderr'
);
RemoveContainerJob::dispatch($this->server->id, $containerName)
->delay(now()->addMinutes(5));
}
private function stop_running_container(bool $force = false)
{
try {
@@ -5016,9 +5048,7 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
// do not remove already running container for PR deployments
} else {
$this->application_deployment_queue->addLogEntry('Deployment failed. Removing the new version of your application.', 'stderr');
$this->execute_remote_command(
["docker rm -f $this->container_name >/dev/null 2>&1", 'hidden' => true, 'ignore_errors' => true]
);
$this->removeContainerWithTimeout($this->container_name);
}
}
}
+49
View File
@@ -0,0 +1,49 @@
<?php
namespace App\Jobs;
use App\Models\Server;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
class RemoveContainerJob implements ShouldBeEncrypted, ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public int $tries = 3;
public int $timeout = 90;
public function __construct(public int $serverId, public string $containerName) {}
public function handle(): void
{
$server = Server::findOrFail($this->serverId);
instant_remote_process(
[dockerRemoveCommandWithTimeout($this->containerName)],
$server,
timeout: 75,
disableMultiplexing: true,
);
}
public function backoff(): array
{
return [300, 900];
}
public function failed(?\Throwable $exception): void
{
Log::warning('Deferred container removal failed', [
'server_id' => $this->serverId,
'container' => $this->containerName,
'error' => $exception?->getMessage(),
]);
}
}
+5
View File
@@ -20,6 +20,9 @@ class Index extends Component
#[Validate('nullable|string|max:255|url')]
public ?string $fqdn = null;
#[Validate('boolean')]
public bool $is_dashboard_force_https_enabled = true;
#[Validate('required|integer|min:1025|max:65535')]
public int $public_port_min;
@@ -68,6 +71,7 @@ class Index extends Component
$this->server = Server::findOrFail(0);
}
$this->fqdn = $this->settings->fqdn;
$this->is_dashboard_force_https_enabled = $this->settings->is_dashboard_force_https_enabled;
$this->public_port_min = $this->settings->public_port_min;
$this->public_port_max = $this->settings->public_port_max;
$this->instance_name = $this->settings->instance_name;
@@ -91,6 +95,7 @@ class Index extends Component
$this->authorize('update', $this->settings);
$this->validate();
$this->settings->fqdn = $this->fqdn ? trim($this->fqdn) : $this->fqdn;
$this->settings->is_dashboard_force_https_enabled = $this->is_dashboard_force_https_enabled;
$this->settings->public_port_min = $this->public_port_min;
$this->settings->public_port_max = $this->public_port_max;
$this->settings->instance_name = $this->instance_name;
+6
View File
@@ -9,6 +9,10 @@ use Spatie\Url\Url;
class InstanceSettings extends Model
{
protected $attributes = [
'is_dashboard_force_https_enabled' => true,
];
protected $fillable = [
'public_ipv4',
'public_ipv6',
@@ -51,6 +55,7 @@ class InstanceSettings extends Model
'webhook_allow_localhost',
'avatar_storage_type',
'avatar_s3_storage_id',
'is_dashboard_force_https_enabled',
];
protected $hidden = [
@@ -89,6 +94,7 @@ class InstanceSettings extends Model
'is_mcp_server_enabled' => 'boolean',
'webhook_allowed_internal_hosts' => 'array',
'webhook_allow_localhost' => 'boolean',
'is_dashboard_force_https_enabled' => 'boolean',
];
protected static function booted(): void
+21 -4
View File
@@ -731,9 +731,7 @@ class Server extends BaseModel
];
if ($schema === 'https') {
$traefik_dynamic_conf['http']['routers']['coolify-http']['middlewares'] = [
0 => 'redirect-to-https',
];
$traefik_dynamic_conf['http']['routers']['coolify-http']['middlewares'] = $this->dashboardHttpMiddlewares($settings);
$traefik_dynamic_conf['http']['routers']['coolify-https'] = [
'middlewares' => [
@@ -792,8 +790,9 @@ class Server extends BaseModel
$url = Url::fromString($settings->fqdn);
$host = $url->getHost();
$schema = $url->getScheme();
$siteAddress = $this->dashboardCaddySiteAddress($settings, $schema, $host);
$caddy_file = "
$schema://$host {
$siteAddress {
encode zstd gzip
handle /app/* {
reverse_proxy coolify-realtime:6001
@@ -819,6 +818,24 @@ $schema://$host {
], $this);
}
public function dashboardHttpMiddlewares(InstanceSettings $settings): array
{
if ($settings->is_dashboard_force_https_enabled) {
return ['redirect-to-https'];
}
return ['gzip'];
}
public function dashboardCaddySiteAddress(InstanceSettings $settings, string $schema, string $host): string
{
if ($schema === 'https' && ! $settings->is_dashboard_force_https_enabled) {
return "http://{$host}, https://{$host}";
}
return "{$schema}://{$host}";
}
public function proxyPath()
{
$base_path = config('constants.coolify.base_config_path');
+9
View File
@@ -263,6 +263,15 @@ function dockerStopCommand(int $timeout, string $containers, Server|string|null
return $command;
}
function dockerRemoveCommandWithTimeout(string $container, int $timeout = 60, int $killAfter = 10): string
{
$container = escapeShellValue($container);
$script = "if command -v timeout >/dev/null 2>&1; then timeout -k {$killAfter}s {$timeout}s docker rm -f {$container}; exit_code=\$?; else exit_code=124; fi; if [ \"\$exit_code\" -eq 124 ]; then echo '__COOLIFY_CONTAINER_REMOVE_TIMEOUT__'; fi; exit \$exit_code";
return 'bash -c '.escapeShellValue($script);
}
function escapeShellValue(string $value): string
{
return "'".str_replace("'", "'\\''", $value)."'";
@@ -0,0 +1,28 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('instance_settings', function (Blueprint $table) {
$table->boolean('is_dashboard_force_https_enabled')->default(true);
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('instance_settings', function (Blueprint $table) {
$table->dropColumn('is_dashboard_force_https_enabled');
});
}
};
@@ -11,12 +11,27 @@
targets="fqdn,instance_name,public_ipv4,public_ipv6,dev_helper_version" />
<x-application.settings-section title="General">
<div class="grid gap-4 lg:grid-cols-2">
<div class="lg:col-span-2">
<div @class([
'lg:col-span-2' => !str_starts_with(strtolower($fqdn ?? ''), 'https://'),
])>
<x-forms.input canGate="update" :canResource="$settings" id="fqdn" label="URL"
helper="Enter the full URL of the instance (for example, https://dashboard.example.com).<br><br><span class='text-coollabs dark:text-warning'>Important:</span> Include <b>https://</b> to secure the dashboard with HTTPS."
placeholder="https://coolify.yourdomain.com" />
</div>
@if (str_starts_with(strtolower($fqdn ?? ''), 'https://'))
<div>
<x-forms.listbox canGate="update" :canResource="$settings"
id="is_dashboard_force_https_enabled" label="Redirect HTTP to HTTPS"
onChange="submit"
helper="Disable only when Cloudflare Tunnel or another proxy connects to Coolify over HTTP. Keep enabled when Cloudflare uses Full or Full (Strict) SSL."
:options="[
['value' => true, 'label' => 'Enabled'],
['value' => false, 'label' => 'Disabled'],
]" />
</div>
@endif
<x-forms.input canGate="update" :canResource="$settings" id="instance_name" label="Name"
placeholder="Coolify" helper="Custom name for this Coolify instance." />
@@ -0,0 +1,94 @@
<?php
use App\Livewire\Settings\Index;
use App\Models\InstanceSettings;
use App\Models\Server;
use App\Models\Team;
use App\Models\User;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Once;
use Livewire\Livewire;
uses(RefreshDatabase::class);
test('dashboard proxy keeps existing HTTPS redirect behavior by default', function () {
$settings = new InstanceSettings;
$server = new Server;
expect($settings->is_dashboard_force_https_enabled)->toBeTrue()
->and($server->dashboardHttpMiddlewares($settings))->toBe(['redirect-to-https'])
->and($server->dashboardCaddySiteAddress($settings, 'https', 'dashboard.example.com'))
->toBe('https://dashboard.example.com');
});
test('dashboard proxy accepts HTTP and HTTPS when its redirect is disabled', function () {
$settings = new InstanceSettings(['is_dashboard_force_https_enabled' => false]);
$server = new Server;
expect($server->dashboardHttpMiddlewares($settings))->toBe(['gzip'])
->and($server->dashboardCaddySiteAddress($settings, 'https', 'dashboard.example.com'))
->toBe('http://dashboard.example.com, https://dashboard.example.com')
->and($server->dashboardCaddySiteAddress($settings, 'http', 'dashboard.example.com'))
->toBe('http://dashboard.example.com');
});
test('instance administrators can configure the dashboard HTTPS redirect', function () {
$rootTeam = Team::find(0) ?? Team::factory()->create(['id' => 0]);
Server::factory()->create(['id' => 0, 'team_id' => $rootTeam->id]);
$settings = InstanceSettings::forceCreate([
'id' => 0,
'fqdn' => 'https://dashboard.example.com',
]);
Once::flush();
$user = User::factory()->create();
$rootTeam->members()->attach($user->id, ['role' => 'admin']);
$this->actingAs($user);
session(['currentTeam' => ['id' => $rootTeam->id]]);
Livewire::test(Index::class)
->assertSet('is_dashboard_force_https_enabled', true)
->assertSee('Redirect HTTP to HTTPS')
->assertSee('Keep enabled when Cloudflare uses Full or Full (Strict) SSL.')
->set('is_dashboard_force_https_enabled', false)
->call('instantSave')
->assertHasNoErrors();
expect($settings->fresh()->is_dashboard_force_https_enabled)->toBeFalse();
});
test('dashboard HTTPS redirect saves immediately when changed', function () {
$contents = file_get_contents(resource_path('views/livewire/settings/index.blade.php'));
expect($contents)
->toMatch('/id="is_dashboard_force_https_enabled"[\s\S]*?onChange="submit"/')
->and($contents)->not->toContain('targets="fqdn,is_dashboard_force_https_enabled');
});
test('dashboard HTTPS redirect is next to the URL on desktop', function () {
$contents = file_get_contents(resource_path('views/livewire/settings/index.blade.php'));
expect($contents)
->toContain("'lg:col-span-2' => !str_starts_with")
->not->toContain('<div class="lg:col-span-2 max-w-md">');
});
test('dashboard HTTPS redirect control is hidden for an HTTP URL', function () {
$rootTeam = Team::find(0) ?? Team::factory()->create(['id' => 0]);
Server::factory()->create(['id' => 0, 'team_id' => $rootTeam->id]);
InstanceSettings::forceCreate([
'id' => 0,
'fqdn' => 'http://dashboard.example.com',
]);
Once::flush();
$user = User::factory()->create();
$rootTeam->members()->attach($user->id, ['role' => 'admin']);
$this->actingAs($user);
session(['currentTeam' => ['id' => $rootTeam->id]]);
Livewire::test(Index::class)
->assertDontSee('Redirect HTTP to HTTPS');
});
@@ -0,0 +1,78 @@
<?php
use App\Jobs\RemoveContainerJob;
use App\Models\Server;
use Symfony\Component\Process\Process;
it('bounds forced container removal and reports a timeout marker', function () {
expect(dockerRemoveCommandWithTimeout('container name'))
->toStartWith("bash -c '")
->toContain('timeout -k 10s 60s docker rm -f')
->toContain('__COOLIFY_CONTAINER_REMOVE_TIMEOUT__');
});
it('uses timeout syntax supported by coreutils and busybox', function () {
expect(dockerRemoveCommandWithTimeout('container-name'))
->toContain('command -v timeout')
->toContain('timeout -k 10s 60s')
->not->toContain('--kill-after');
});
it('preserves bounded cleanup when commands are adapted for non-root servers', function () {
$server = Mockery::mock(Server::class)->makePartial();
$server->shouldReceive('getAttribute')->with('user')->andReturn('ubuntu');
$server->shouldReceive('setAttribute')->andReturnSelf();
$server->user = 'ubuntu';
$commands = parseCommandsByLineForSudo(
collect([dockerRemoveCommandWithTimeout('container-name')]),
$server
);
$command = $commands[0];
expect($command)
->toStartWith("sudo bash -c '")
->toContain('timeout -k 10s 60s docker rm -f')
->toContain('__COOLIFY_CONTAINER_REMOVE_TIMEOUT__');
});
it('escapes container names in bounded removal commands', function () {
$containerName = "container'; reboot; '";
$directory = sys_get_temp_dir().'/coolify-docker-remove-'.bin2hex(random_bytes(4));
$captureFile = $directory.'/arguments';
mkdir($directory);
file_put_contents($directory.'/docker', "#!/bin/sh\nprintf '%s\\n' \"\$@\" > \"\$CAPTURE_FILE\"\n");
chmod($directory.'/docker', 0755);
$process = new Process(['/bin/sh', '-c', dockerRemoveCommandWithTimeout($containerName)], env: [
'PATH' => $directory.':'.getenv('PATH'),
'CAPTURE_FILE' => $captureFile,
]);
$process->run();
expect($process->isSuccessful())->toBeTrue()
->and(file_get_contents($captureFile))->toBe("rm\n-f\n{$containerName}\n");
unlink($captureFile);
unlink($directory.'/docker');
rmdir($directory);
});
it('configures deferred removal attempts to outlive the shell timeout', function () {
$job = new RemoveContainerJob(123, 'container-name');
expect($job->serverId)->toBe(123)
->and($job->containerName)->toBe('container-name')
->and($job->timeout)->toBeGreaterThan(60)
->and($job->tries)->toBeGreaterThan(1);
});
it('continues deployments and schedules deferred cleanup after a removal timeout', function () {
$source = file_get_contents(dirname(__DIR__, 2).'/app/Jobs/ApplicationDeploymentJob.php');
expect($source)
->toContain('dockerRemoveCommandWithTimeout($containerName)')
->toContain('timed out after 60 seconds. The deployment will continue')
->toContain('RemoveContainerJob::dispatch($this->server->id, $containerName)')
->toContain('->delay(now()->addMinutes(5))');
});