mirror of
https://github.com/coollabsio/coolify.git
synced 2026-08-24 10:05:47 -05:00
fix(deployments): defer container cleanup and configure dashboard HTTPS
This commit is contained in:
@@ -52,6 +52,8 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
|
||||
|
||||
private const RAILPACK_GENERATED_CONFIG_PATH = '.coolify/railpack.generated.json';
|
||||
|
||||
private const CONTAINER_REMOVE_TIMEOUT_MARKER = '__COOLIFY_CONTAINER_REMOVE_TIMEOUT__';
|
||||
|
||||
private const DOCKER_CLIENT_ENV_KEYS = [
|
||||
'BUILDKIT_HOST',
|
||||
'BUILDX_BUILDER',
|
||||
@@ -3977,15 +3979,45 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
|
||||
);
|
||||
} else {
|
||||
$this->execute_remote_command(
|
||||
[dockerStopCommand($timeout, $containerName, $this->server), 'hidden' => true, 'ignore_errors' => true],
|
||||
["docker rm -f $containerName", 'hidden' => true, 'ignore_errors' => true]
|
||||
[dockerStopCommand($timeout, $containerName, $this->server), 'hidden' => true, 'ignore_errors' => true]
|
||||
);
|
||||
$this->removeContainerWithTimeout($containerName);
|
||||
}
|
||||
} catch (Exception $error) {
|
||||
$this->application_deployment_queue->addLogEntry("Error stopping container $containerName: ".$error->getMessage(), 'stderr');
|
||||
}
|
||||
}
|
||||
|
||||
private function removeContainerWithTimeout(string $containerName): void
|
||||
{
|
||||
$outputKey = 'container_remove_'.md5($containerName);
|
||||
|
||||
$this->execute_remote_command([
|
||||
dockerRemoveCommandWithTimeout($containerName),
|
||||
'hidden' => true,
|
||||
'ignore_errors' => true,
|
||||
'save' => $outputKey,
|
||||
'append' => false,
|
||||
]);
|
||||
|
||||
if (! isset($this->saved_outputs)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$output = (string) $this->saved_outputs->get($outputKey, '');
|
||||
if (! str_contains($output, self::CONTAINER_REMOVE_TIMEOUT_MARKER)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$this->application_deployment_queue->addLogEntry(
|
||||
"Warning: Removing container {$containerName} timed out after 60 seconds. The deployment will continue and cleanup will be retried in 5 minutes.",
|
||||
'stderr'
|
||||
);
|
||||
|
||||
RemoveContainerJob::dispatch($this->server->id, $containerName)
|
||||
->delay(now()->addMinutes(5));
|
||||
}
|
||||
|
||||
private function stop_running_container(bool $force = false)
|
||||
{
|
||||
try {
|
||||
@@ -5016,9 +5048,7 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
|
||||
// do not remove already running container for PR deployments
|
||||
} else {
|
||||
$this->application_deployment_queue->addLogEntry('Deployment failed. Removing the new version of your application.', 'stderr');
|
||||
$this->execute_remote_command(
|
||||
["docker rm -f $this->container_name >/dev/null 2>&1", 'hidden' => true, 'ignore_errors' => true]
|
||||
);
|
||||
$this->removeContainerWithTimeout($this->container_name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
<?php
|
||||
|
||||
namespace App\Jobs;
|
||||
|
||||
use App\Models\Server;
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
class RemoveContainerJob implements ShouldBeEncrypted, ShouldQueue
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
public int $tries = 3;
|
||||
|
||||
public int $timeout = 90;
|
||||
|
||||
public function __construct(public int $serverId, public string $containerName) {}
|
||||
|
||||
public function handle(): void
|
||||
{
|
||||
$server = Server::findOrFail($this->serverId);
|
||||
|
||||
instant_remote_process(
|
||||
[dockerRemoveCommandWithTimeout($this->containerName)],
|
||||
$server,
|
||||
timeout: 75,
|
||||
disableMultiplexing: true,
|
||||
);
|
||||
}
|
||||
|
||||
public function backoff(): array
|
||||
{
|
||||
return [300, 900];
|
||||
}
|
||||
|
||||
public function failed(?\Throwable $exception): void
|
||||
{
|
||||
Log::warning('Deferred container removal failed', [
|
||||
'server_id' => $this->serverId,
|
||||
'container' => $this->containerName,
|
||||
'error' => $exception?->getMessage(),
|
||||
]);
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,9 @@ class Index extends Component
|
||||
#[Validate('nullable|string|max:255|url')]
|
||||
public ?string $fqdn = null;
|
||||
|
||||
#[Validate('boolean')]
|
||||
public bool $is_dashboard_force_https_enabled = true;
|
||||
|
||||
#[Validate('required|integer|min:1025|max:65535')]
|
||||
public int $public_port_min;
|
||||
|
||||
@@ -68,6 +71,7 @@ class Index extends Component
|
||||
$this->server = Server::findOrFail(0);
|
||||
}
|
||||
$this->fqdn = $this->settings->fqdn;
|
||||
$this->is_dashboard_force_https_enabled = $this->settings->is_dashboard_force_https_enabled;
|
||||
$this->public_port_min = $this->settings->public_port_min;
|
||||
$this->public_port_max = $this->settings->public_port_max;
|
||||
$this->instance_name = $this->settings->instance_name;
|
||||
@@ -91,6 +95,7 @@ class Index extends Component
|
||||
$this->authorize('update', $this->settings);
|
||||
$this->validate();
|
||||
$this->settings->fqdn = $this->fqdn ? trim($this->fqdn) : $this->fqdn;
|
||||
$this->settings->is_dashboard_force_https_enabled = $this->is_dashboard_force_https_enabled;
|
||||
$this->settings->public_port_min = $this->public_port_min;
|
||||
$this->settings->public_port_max = $this->public_port_max;
|
||||
$this->settings->instance_name = $this->instance_name;
|
||||
|
||||
@@ -9,6 +9,10 @@ use Spatie\Url\Url;
|
||||
|
||||
class InstanceSettings extends Model
|
||||
{
|
||||
protected $attributes = [
|
||||
'is_dashboard_force_https_enabled' => true,
|
||||
];
|
||||
|
||||
protected $fillable = [
|
||||
'public_ipv4',
|
||||
'public_ipv6',
|
||||
@@ -51,6 +55,7 @@ class InstanceSettings extends Model
|
||||
'webhook_allow_localhost',
|
||||
'avatar_storage_type',
|
||||
'avatar_s3_storage_id',
|
||||
'is_dashboard_force_https_enabled',
|
||||
];
|
||||
|
||||
protected $hidden = [
|
||||
@@ -89,6 +94,7 @@ class InstanceSettings extends Model
|
||||
'is_mcp_server_enabled' => 'boolean',
|
||||
'webhook_allowed_internal_hosts' => 'array',
|
||||
'webhook_allow_localhost' => 'boolean',
|
||||
'is_dashboard_force_https_enabled' => 'boolean',
|
||||
];
|
||||
|
||||
protected static function booted(): void
|
||||
|
||||
+21
-4
@@ -731,9 +731,7 @@ class Server extends BaseModel
|
||||
];
|
||||
|
||||
if ($schema === 'https') {
|
||||
$traefik_dynamic_conf['http']['routers']['coolify-http']['middlewares'] = [
|
||||
0 => 'redirect-to-https',
|
||||
];
|
||||
$traefik_dynamic_conf['http']['routers']['coolify-http']['middlewares'] = $this->dashboardHttpMiddlewares($settings);
|
||||
|
||||
$traefik_dynamic_conf['http']['routers']['coolify-https'] = [
|
||||
'middlewares' => [
|
||||
@@ -792,8 +790,9 @@ class Server extends BaseModel
|
||||
$url = Url::fromString($settings->fqdn);
|
||||
$host = $url->getHost();
|
||||
$schema = $url->getScheme();
|
||||
$siteAddress = $this->dashboardCaddySiteAddress($settings, $schema, $host);
|
||||
$caddy_file = "
|
||||
$schema://$host {
|
||||
$siteAddress {
|
||||
encode zstd gzip
|
||||
handle /app/* {
|
||||
reverse_proxy coolify-realtime:6001
|
||||
@@ -819,6 +818,24 @@ $schema://$host {
|
||||
], $this);
|
||||
}
|
||||
|
||||
public function dashboardHttpMiddlewares(InstanceSettings $settings): array
|
||||
{
|
||||
if ($settings->is_dashboard_force_https_enabled) {
|
||||
return ['redirect-to-https'];
|
||||
}
|
||||
|
||||
return ['gzip'];
|
||||
}
|
||||
|
||||
public function dashboardCaddySiteAddress(InstanceSettings $settings, string $schema, string $host): string
|
||||
{
|
||||
if ($schema === 'https' && ! $settings->is_dashboard_force_https_enabled) {
|
||||
return "http://{$host}, https://{$host}";
|
||||
}
|
||||
|
||||
return "{$schema}://{$host}";
|
||||
}
|
||||
|
||||
public function proxyPath()
|
||||
{
|
||||
$base_path = config('constants.coolify.base_config_path');
|
||||
|
||||
@@ -263,6 +263,15 @@ function dockerStopCommand(int $timeout, string $containers, Server|string|null
|
||||
|
||||
return $command;
|
||||
}
|
||||
|
||||
function dockerRemoveCommandWithTimeout(string $container, int $timeout = 60, int $killAfter = 10): string
|
||||
{
|
||||
$container = escapeShellValue($container);
|
||||
$script = "if command -v timeout >/dev/null 2>&1; then timeout -k {$killAfter}s {$timeout}s docker rm -f {$container}; exit_code=\$?; else exit_code=124; fi; if [ \"\$exit_code\" -eq 124 ]; then echo '__COOLIFY_CONTAINER_REMOVE_TIMEOUT__'; fi; exit \$exit_code";
|
||||
|
||||
return 'bash -c '.escapeShellValue($script);
|
||||
}
|
||||
|
||||
function escapeShellValue(string $value): string
|
||||
{
|
||||
return "'".str_replace("'", "'\\''", $value)."'";
|
||||
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
<?php
|
||||
|
||||
use Illuminate\Database\Migrations\Migration;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
use Illuminate\Support\Facades\Schema;
|
||||
|
||||
return new class extends Migration
|
||||
{
|
||||
/**
|
||||
* Run the migrations.
|
||||
*/
|
||||
public function up(): void
|
||||
{
|
||||
Schema::table('instance_settings', function (Blueprint $table) {
|
||||
$table->boolean('is_dashboard_force_https_enabled')->default(true);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Reverse the migrations.
|
||||
*/
|
||||
public function down(): void
|
||||
{
|
||||
Schema::table('instance_settings', function (Blueprint $table) {
|
||||
$table->dropColumn('is_dashboard_force_https_enabled');
|
||||
});
|
||||
}
|
||||
};
|
||||
@@ -11,12 +11,27 @@
|
||||
targets="fqdn,instance_name,public_ipv4,public_ipv6,dev_helper_version" />
|
||||
<x-application.settings-section title="General">
|
||||
<div class="grid gap-4 lg:grid-cols-2">
|
||||
<div class="lg:col-span-2">
|
||||
<div @class([
|
||||
'lg:col-span-2' => !str_starts_with(strtolower($fqdn ?? ''), 'https://'),
|
||||
])>
|
||||
<x-forms.input canGate="update" :canResource="$settings" id="fqdn" label="URL"
|
||||
helper="Enter the full URL of the instance (for example, https://dashboard.example.com).<br><br><span class='text-coollabs dark:text-warning'>Important:</span> Include <b>https://</b> to secure the dashboard with HTTPS."
|
||||
placeholder="https://coolify.yourdomain.com" />
|
||||
</div>
|
||||
|
||||
@if (str_starts_with(strtolower($fqdn ?? ''), 'https://'))
|
||||
<div>
|
||||
<x-forms.listbox canGate="update" :canResource="$settings"
|
||||
id="is_dashboard_force_https_enabled" label="Redirect HTTP to HTTPS"
|
||||
onChange="submit"
|
||||
helper="Disable only when Cloudflare Tunnel or another proxy connects to Coolify over HTTP. Keep enabled when Cloudflare uses Full or Full (Strict) SSL."
|
||||
:options="[
|
||||
['value' => true, 'label' => 'Enabled'],
|
||||
['value' => false, 'label' => 'Disabled'],
|
||||
]" />
|
||||
</div>
|
||||
@endif
|
||||
|
||||
<x-forms.input canGate="update" :canResource="$settings" id="instance_name" label="Name"
|
||||
placeholder="Coolify" helper="Custom name for this Coolify instance." />
|
||||
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
<?php
|
||||
|
||||
use App\Livewire\Settings\Index;
|
||||
use App\Models\InstanceSettings;
|
||||
use App\Models\Server;
|
||||
use App\Models\Team;
|
||||
use App\Models\User;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
use Illuminate\Support\Once;
|
||||
use Livewire\Livewire;
|
||||
|
||||
uses(RefreshDatabase::class);
|
||||
|
||||
test('dashboard proxy keeps existing HTTPS redirect behavior by default', function () {
|
||||
$settings = new InstanceSettings;
|
||||
$server = new Server;
|
||||
|
||||
expect($settings->is_dashboard_force_https_enabled)->toBeTrue()
|
||||
->and($server->dashboardHttpMiddlewares($settings))->toBe(['redirect-to-https'])
|
||||
->and($server->dashboardCaddySiteAddress($settings, 'https', 'dashboard.example.com'))
|
||||
->toBe('https://dashboard.example.com');
|
||||
});
|
||||
|
||||
test('dashboard proxy accepts HTTP and HTTPS when its redirect is disabled', function () {
|
||||
$settings = new InstanceSettings(['is_dashboard_force_https_enabled' => false]);
|
||||
$server = new Server;
|
||||
|
||||
expect($server->dashboardHttpMiddlewares($settings))->toBe(['gzip'])
|
||||
->and($server->dashboardCaddySiteAddress($settings, 'https', 'dashboard.example.com'))
|
||||
->toBe('http://dashboard.example.com, https://dashboard.example.com')
|
||||
->and($server->dashboardCaddySiteAddress($settings, 'http', 'dashboard.example.com'))
|
||||
->toBe('http://dashboard.example.com');
|
||||
});
|
||||
|
||||
test('instance administrators can configure the dashboard HTTPS redirect', function () {
|
||||
$rootTeam = Team::find(0) ?? Team::factory()->create(['id' => 0]);
|
||||
Server::factory()->create(['id' => 0, 'team_id' => $rootTeam->id]);
|
||||
$settings = InstanceSettings::forceCreate([
|
||||
'id' => 0,
|
||||
'fqdn' => 'https://dashboard.example.com',
|
||||
]);
|
||||
Once::flush();
|
||||
|
||||
$user = User::factory()->create();
|
||||
$rootTeam->members()->attach($user->id, ['role' => 'admin']);
|
||||
|
||||
$this->actingAs($user);
|
||||
session(['currentTeam' => ['id' => $rootTeam->id]]);
|
||||
|
||||
Livewire::test(Index::class)
|
||||
->assertSet('is_dashboard_force_https_enabled', true)
|
||||
->assertSee('Redirect HTTP to HTTPS')
|
||||
->assertSee('Keep enabled when Cloudflare uses Full or Full (Strict) SSL.')
|
||||
->set('is_dashboard_force_https_enabled', false)
|
||||
->call('instantSave')
|
||||
->assertHasNoErrors();
|
||||
|
||||
expect($settings->fresh()->is_dashboard_force_https_enabled)->toBeFalse();
|
||||
});
|
||||
|
||||
test('dashboard HTTPS redirect saves immediately when changed', function () {
|
||||
$contents = file_get_contents(resource_path('views/livewire/settings/index.blade.php'));
|
||||
|
||||
expect($contents)
|
||||
->toMatch('/id="is_dashboard_force_https_enabled"[\s\S]*?onChange="submit"/')
|
||||
->and($contents)->not->toContain('targets="fqdn,is_dashboard_force_https_enabled');
|
||||
});
|
||||
|
||||
test('dashboard HTTPS redirect is next to the URL on desktop', function () {
|
||||
$contents = file_get_contents(resource_path('views/livewire/settings/index.blade.php'));
|
||||
|
||||
expect($contents)
|
||||
->toContain("'lg:col-span-2' => !str_starts_with")
|
||||
->not->toContain('<div class="lg:col-span-2 max-w-md">');
|
||||
});
|
||||
|
||||
test('dashboard HTTPS redirect control is hidden for an HTTP URL', function () {
|
||||
$rootTeam = Team::find(0) ?? Team::factory()->create(['id' => 0]);
|
||||
Server::factory()->create(['id' => 0, 'team_id' => $rootTeam->id]);
|
||||
InstanceSettings::forceCreate([
|
||||
'id' => 0,
|
||||
'fqdn' => 'http://dashboard.example.com',
|
||||
]);
|
||||
Once::flush();
|
||||
|
||||
$user = User::factory()->create();
|
||||
$rootTeam->members()->attach($user->id, ['role' => 'admin']);
|
||||
|
||||
$this->actingAs($user);
|
||||
session(['currentTeam' => ['id' => $rootTeam->id]]);
|
||||
|
||||
Livewire::test(Index::class)
|
||||
->assertDontSee('Redirect HTTP to HTTPS');
|
||||
});
|
||||
@@ -0,0 +1,78 @@
|
||||
<?php
|
||||
|
||||
use App\Jobs\RemoveContainerJob;
|
||||
use App\Models\Server;
|
||||
use Symfony\Component\Process\Process;
|
||||
|
||||
it('bounds forced container removal and reports a timeout marker', function () {
|
||||
expect(dockerRemoveCommandWithTimeout('container name'))
|
||||
->toStartWith("bash -c '")
|
||||
->toContain('timeout -k 10s 60s docker rm -f')
|
||||
->toContain('__COOLIFY_CONTAINER_REMOVE_TIMEOUT__');
|
||||
});
|
||||
|
||||
it('uses timeout syntax supported by coreutils and busybox', function () {
|
||||
expect(dockerRemoveCommandWithTimeout('container-name'))
|
||||
->toContain('command -v timeout')
|
||||
->toContain('timeout -k 10s 60s')
|
||||
->not->toContain('--kill-after');
|
||||
});
|
||||
|
||||
it('preserves bounded cleanup when commands are adapted for non-root servers', function () {
|
||||
$server = Mockery::mock(Server::class)->makePartial();
|
||||
$server->shouldReceive('getAttribute')->with('user')->andReturn('ubuntu');
|
||||
$server->shouldReceive('setAttribute')->andReturnSelf();
|
||||
$server->user = 'ubuntu';
|
||||
|
||||
$commands = parseCommandsByLineForSudo(
|
||||
collect([dockerRemoveCommandWithTimeout('container-name')]),
|
||||
$server
|
||||
);
|
||||
$command = $commands[0];
|
||||
|
||||
expect($command)
|
||||
->toStartWith("sudo bash -c '")
|
||||
->toContain('timeout -k 10s 60s docker rm -f')
|
||||
->toContain('__COOLIFY_CONTAINER_REMOVE_TIMEOUT__');
|
||||
});
|
||||
|
||||
it('escapes container names in bounded removal commands', function () {
|
||||
$containerName = "container'; reboot; '";
|
||||
$directory = sys_get_temp_dir().'/coolify-docker-remove-'.bin2hex(random_bytes(4));
|
||||
$captureFile = $directory.'/arguments';
|
||||
mkdir($directory);
|
||||
file_put_contents($directory.'/docker', "#!/bin/sh\nprintf '%s\\n' \"\$@\" > \"\$CAPTURE_FILE\"\n");
|
||||
chmod($directory.'/docker', 0755);
|
||||
|
||||
$process = new Process(['/bin/sh', '-c', dockerRemoveCommandWithTimeout($containerName)], env: [
|
||||
'PATH' => $directory.':'.getenv('PATH'),
|
||||
'CAPTURE_FILE' => $captureFile,
|
||||
]);
|
||||
$process->run();
|
||||
|
||||
expect($process->isSuccessful())->toBeTrue()
|
||||
->and(file_get_contents($captureFile))->toBe("rm\n-f\n{$containerName}\n");
|
||||
|
||||
unlink($captureFile);
|
||||
unlink($directory.'/docker');
|
||||
rmdir($directory);
|
||||
});
|
||||
|
||||
it('configures deferred removal attempts to outlive the shell timeout', function () {
|
||||
$job = new RemoveContainerJob(123, 'container-name');
|
||||
|
||||
expect($job->serverId)->toBe(123)
|
||||
->and($job->containerName)->toBe('container-name')
|
||||
->and($job->timeout)->toBeGreaterThan(60)
|
||||
->and($job->tries)->toBeGreaterThan(1);
|
||||
});
|
||||
|
||||
it('continues deployments and schedules deferred cleanup after a removal timeout', function () {
|
||||
$source = file_get_contents(dirname(__DIR__, 2).'/app/Jobs/ApplicationDeploymentJob.php');
|
||||
|
||||
expect($source)
|
||||
->toContain('dockerRemoveCommandWithTimeout($containerName)')
|
||||
->toContain('timed out after 60 seconds. The deployment will continue')
|
||||
->toContain('RemoveContainerJob::dispatch($this->server->id, $containerName)')
|
||||
->toContain('->delay(now()->addMinutes(5))');
|
||||
});
|
||||
Reference in New Issue
Block a user