From a3712f8902f205208b253117f956577d3d951560 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Sat, 24 Oct 2026 11:16:46 +0200 Subject: [PATCH] fix(deploy): stop 502 errors during rolling updates Traefik and Caddy routes of applications now retry a failed connection on another container, and the rolling update waits a configurable drain time (default 5 seconds) after the new container is healthy before it stops the old containers. The wait is skipped when no old container runs or when the proxy labels changed, because the proxy drops a route that two containers define differently. Fixes #8627 --- .ai/lessons.md | 1 + .../Api/ApplicationsController.php | 7 + app/Jobs/ApplicationDeploymentJob.php | 41 ++++ app/Livewire/Project/Application/Advanced.php | 27 +++ app/Models/ApplicationSetting.php | 8 + bootstrap/helpers/api.php | 2 + bootstrap/helpers/constants.php | 2 + bootstrap/helpers/docker.php | 21 +- ..._drain_seconds_to_application_settings.php | 31 +++ docs/server-transfer-full-coverage.md | 2 +- openapi.json | 40 ++++ openapi.yaml | 33 +++ .../project/application/advanced.blade.php | 7 + .../Api/ApplicationSettingsApiTest.php | 13 ++ tests/Feature/RollingUpdateProxyDrainTest.php | 214 ++++++++++++++++++ 15 files changed, 446 insertions(+), 3 deletions(-) create mode 100644 database/migrations/2026_10_23_224202_add_rolling_update_drain_seconds_to_application_settings.php create mode 100644 tests/Feature/RollingUpdateProxyDrainTest.php diff --git a/.ai/lessons.md b/.ai/lessons.md index 00dc3cbd89..79f47a68e6 100644 --- a/.ai/lessons.md +++ b/.ai/lessons.md @@ -25,6 +25,7 @@ - The helper image runs `docker compose` for deployments and for the Coolify upgrade itself. Before you bump its Docker CLI or Compose, test them against the oldest supported Docker Engine (for example a `docker:-dind` container); newer clients can reject data from older daemons. - Do not symlink `vendor` into another worktree. Composer resolves the base path from the real `vendor` path, so the tests load the app classes of the other checkout. Run `composer install --ignore-platform-req=ext-zip` in the worktree. - To assert that the unsaved-changes bar shows, check `el.classList.contains('is-dirty')`. Playwright treats the hidden bar (`opacity-0`) as visible, and its class attribute always contains the text `is-dirty` from Tailwind variants. +- For load tests against a dev VM, use your own instance (`COOLIFY_DEV_INSTANCE= COOLIFY_DEV_SIDE_BY_SIDE=1 ./scripts/dev start`); other sessions can reseed the localhost VM of a shared instance. The 1 GiB VM drops packets when its conntrack table is full (`dmesg`: `nf_conntrack: table full`), so raise `net.netfilter.nf_conntrack_max` on it and run an idle load first to prove a clean baseline. - Do not call `sleep()` in a browser test. The app server runs in the test process, so `sleep()` stops it; use `$page->wait()`. - `Queue::fake()` also captures `dispatchSync()` of a `ShouldQueue` job, so the job never runs. Fake only the jobs to block (`Queue::fake([SomeJob::class])`). - In tests, turning on `is_metrics_enabled` (and other Sentinel settings) with `save()`/`update()` runs the settings hook, which starts Sentinel over a real SSH connection on the sync queue. Use `saveQuietly()` or `Queue::fake()`. diff --git a/app/Http/Controllers/Api/ApplicationsController.php b/app/Http/Controllers/Api/ApplicationsController.php index 4baeefd913..eb3c02685f 100644 --- a/app/Http/Controllers/Api/ApplicationsController.php +++ b/app/Http/Controllers/Api/ApplicationsController.php @@ -56,6 +56,7 @@ class ApplicationsController extends Controller 'include_coolify_default_variables', 'is_pr_deployments_public_enabled', 'stop_grace_period', + 'rolling_update_drain_seconds', 'docker_images_to_keep', 'is_gzip_enabled', 'is_stripprefix_enabled', @@ -468,6 +469,7 @@ class ApplicationsController extends Controller 'include_coolify_default_variables' => ['type' => 'boolean', 'default' => true, 'description' => 'Add the predefined COOLIFY_* variables (COOLIFY_URL, COOLIFY_FQDN, ...) to the environment. Requires team admin permissions. Redeploy to apply it.'], 'is_pr_deployments_public_enabled' => ['type' => 'boolean', 'description' => 'Make pull request deployments public.'], 'stop_grace_period' => ['type' => 'integer', 'nullable' => true, 'minimum' => 1, 'maximum' => 3600, 'description' => 'Container stop grace period in seconds.'], + 'rolling_update_drain_seconds' => ['type' => 'integer', 'minimum' => 0, 'maximum' => 300, 'description' => 'Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5.'], 'docker_images_to_keep' => ['type' => 'integer', 'minimum' => 0, 'maximum' => 100, 'description' => 'Number of Docker images to retain.'], 'is_gzip_enabled' => ['type' => 'boolean', 'description' => 'Enable gzip compression.'], 'is_stripprefix_enabled' => ['type' => 'boolean', 'description' => 'Enable path prefix stripping.'], @@ -677,6 +679,7 @@ class ApplicationsController extends Controller 'include_coolify_default_variables' => ['type' => 'boolean', 'default' => true, 'description' => 'Add the predefined COOLIFY_* variables (COOLIFY_URL, COOLIFY_FQDN, ...) to the environment. Requires team admin permissions. Redeploy to apply it.'], 'is_pr_deployments_public_enabled' => ['type' => 'boolean', 'description' => 'Make pull request deployments public.'], 'stop_grace_period' => ['type' => 'integer', 'nullable' => true, 'minimum' => 1, 'maximum' => 3600, 'description' => 'Container stop grace period in seconds.'], + 'rolling_update_drain_seconds' => ['type' => 'integer', 'minimum' => 0, 'maximum' => 300, 'description' => 'Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5.'], 'docker_images_to_keep' => ['type' => 'integer', 'minimum' => 0, 'maximum' => 100, 'description' => 'Number of Docker images to retain.'], 'is_gzip_enabled' => ['type' => 'boolean', 'description' => 'Enable gzip compression.'], 'is_stripprefix_enabled' => ['type' => 'boolean', 'description' => 'Enable path prefix stripping.'], @@ -886,6 +889,7 @@ class ApplicationsController extends Controller 'include_coolify_default_variables' => ['type' => 'boolean', 'default' => true, 'description' => 'Add the predefined COOLIFY_* variables (COOLIFY_URL, COOLIFY_FQDN, ...) to the environment. Requires team admin permissions. Redeploy to apply it.'], 'is_pr_deployments_public_enabled' => ['type' => 'boolean', 'description' => 'Make pull request deployments public.'], 'stop_grace_period' => ['type' => 'integer', 'nullable' => true, 'minimum' => 1, 'maximum' => 3600, 'description' => 'Container stop grace period in seconds.'], + 'rolling_update_drain_seconds' => ['type' => 'integer', 'minimum' => 0, 'maximum' => 300, 'description' => 'Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5.'], 'docker_images_to_keep' => ['type' => 'integer', 'minimum' => 0, 'maximum' => 100, 'description' => 'Number of Docker images to retain.'], 'is_gzip_enabled' => ['type' => 'boolean', 'description' => 'Enable gzip compression.'], 'is_stripprefix_enabled' => ['type' => 'boolean', 'description' => 'Enable path prefix stripping.'], @@ -1065,6 +1069,7 @@ class ApplicationsController extends Controller 'include_coolify_default_variables' => ['type' => 'boolean', 'default' => true, 'description' => 'Add the predefined COOLIFY_* variables (COOLIFY_URL, COOLIFY_FQDN, ...) to the environment. Requires team admin permissions. Redeploy to apply it.'], 'is_pr_deployments_public_enabled' => ['type' => 'boolean', 'description' => 'Make pull request deployments public.'], 'stop_grace_period' => ['type' => 'integer', 'nullable' => true, 'minimum' => 1, 'maximum' => 3600, 'description' => 'Container stop grace period in seconds.'], + 'rolling_update_drain_seconds' => ['type' => 'integer', 'minimum' => 0, 'maximum' => 300, 'description' => 'Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5.'], 'docker_images_to_keep' => ['type' => 'integer', 'minimum' => 0, 'maximum' => 100, 'description' => 'Number of Docker images to retain.'], 'is_gzip_enabled' => ['type' => 'boolean', 'description' => 'Enable gzip compression.'], 'is_stripprefix_enabled' => ['type' => 'boolean', 'description' => 'Enable path prefix stripping.'], @@ -1240,6 +1245,7 @@ class ApplicationsController extends Controller 'include_coolify_default_variables' => ['type' => 'boolean', 'default' => true, 'description' => 'Add the predefined COOLIFY_* variables (COOLIFY_URL, COOLIFY_FQDN, ...) to the environment. Requires team admin permissions. Redeploy to apply it.'], 'is_pr_deployments_public_enabled' => ['type' => 'boolean', 'description' => 'Make pull request deployments public.'], 'stop_grace_period' => ['type' => 'integer', 'nullable' => true, 'minimum' => 1, 'maximum' => 3600, 'description' => 'Container stop grace period in seconds.'], + 'rolling_update_drain_seconds' => ['type' => 'integer', 'minimum' => 0, 'maximum' => 300, 'description' => 'Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5.'], 'docker_images_to_keep' => ['type' => 'integer', 'minimum' => 0, 'maximum' => 100, 'description' => 'Number of Docker images to retain.'], 'is_gzip_enabled' => ['type' => 'boolean', 'description' => 'Enable gzip compression.'], 'is_stripprefix_enabled' => ['type' => 'boolean', 'description' => 'Enable path prefix stripping.'], @@ -3261,6 +3267,7 @@ class ApplicationsController extends Controller 'include_coolify_default_variables' => ['type' => 'boolean', 'default' => true, 'description' => 'Add the predefined COOLIFY_* variables (COOLIFY_URL, COOLIFY_FQDN, ...) to the environment. Requires team admin permissions. Redeploy to apply it.'], 'is_pr_deployments_public_enabled' => ['type' => 'boolean', 'description' => 'Make pull request deployments public.'], 'stop_grace_period' => ['type' => 'integer', 'nullable' => true, 'minimum' => 1, 'maximum' => 3600, 'description' => 'Container stop grace period in seconds.'], + 'rolling_update_drain_seconds' => ['type' => 'integer', 'minimum' => 0, 'maximum' => 300, 'description' => 'Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5.'], 'docker_images_to_keep' => ['type' => 'integer', 'minimum' => 0, 'maximum' => 100, 'description' => 'Number of Docker images to retain.'], 'is_gzip_enabled' => ['type' => 'boolean', 'description' => 'Enable gzip compression.'], 'is_stripprefix_enabled' => ['type' => 'boolean', 'description' => 'Enable path prefix stripping.'], diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index 42b2fc4831..92d5917eaf 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -2709,6 +2709,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue $this->application_deployment_queue->addLogEntry('Rolling update started.'); $this->start_by_compose_file(); $this->health_check(); + $this->wait_for_proxy_to_route_new_container(); $this->stop_running_container(); $this->application_deployment_queue->addLogEntry('Rolling update completed.'); } @@ -2718,6 +2719,46 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue } } + /** + * The proxy adds the new container some seconds after it is healthy. Stopping the old + * container before that sends requests to a closed port (502), see #8627. + */ + private function wait_for_proxy_to_route_new_container(): void + { + $seconds = $this->application->settings->rollingUpdateDrainSeconds(); + if ($seconds === 0 || ! $this->newVersionIsHealthy()) { + return; + } + $oldContainerNames = getCurrentApplicationContainerStatus($this->server, $this->application, $this->pull_request_id) + ->filter(fn ($container) => data_get($container, 'State') === 'running' && data_get($container, 'Names') !== $this->container_name) + ->pluck('Names'); + if ($oldContainerNames->isEmpty()) { + return; + } + if ($this->proxyLabelsChanged($oldContainerNames)) { + // The proxy drops a route that two containers define differently until the old one stops. + $this->application_deployment_queue->addLogEntry('The proxy labels changed, so the old containers stop without waiting.'); + + return; + } + $this->application_deployment_queue->addLogEntry("Waiting {$seconds} seconds for the proxy to send traffic to the new container."); + Sleep::for($seconds)->seconds(); + } + + private function proxyLabelsChanged(Collection $oldContainerNames): bool + { + $containerNames = $oldContainerNames->prepend($this->container_name)->map(fn (string $name) => escapeshellarg($name))->implode(' '); + $output = instant_remote_process(["docker inspect --format '{{json .Config.Labels}}' {$containerNames}"], $this->server, false); + + return collect(preg_split('/\R/', trim((string) $output))) + ->map(fn (string $labels) => collect(json_decode($labels, true) ?: []) + ->filter(fn ($value, string $key) => str_starts_with($key, 'traefik.') || str_starts_with($key, 'caddy')) + ->sortKeys() + ->all()) + ->unique() + ->count() > 1; + } + private function health_check() { try { diff --git a/app/Livewire/Project/Application/Advanced.php b/app/Livewire/Project/Application/Advanced.php index 0075bdddd4..3f3d95573a 100644 --- a/app/Livewire/Project/Application/Advanced.php +++ b/app/Livewire/Project/Application/Advanced.php @@ -68,6 +68,8 @@ class Advanced extends Component #[Validate(['string', 'nullable'])] public ?string $stopGracePeriod = null; + public ?int $rollingUpdateDrainSeconds = DEFAULT_ROLLING_UPDATE_DRAIN_SECONDS; + #[Validate(['boolean'])] public bool $isBuildServerEnabled = false; @@ -170,6 +172,7 @@ class Advanced extends Component // Load stop_grace_period separately since it has its own save handler // Convert null to empty string to prevent dirty detection issues $this->stopGracePeriod = $this->application->settings->stop_grace_period ?? ''; + $this->rollingUpdateDrainSeconds = $this->application->settings->rollingUpdateDrainSeconds(); } private function syncCertificateResolverFromModel(): void @@ -432,6 +435,30 @@ class Advanced extends Component } } + public function saveRollingUpdateDrainSeconds() + { + try { + $this->authorize('update', $this->application); + + $validated = $this->validate( + ['rollingUpdateDrainSeconds' => ['required', 'integer', 'min:0', 'max:'.MAX_ROLLING_UPDATE_DRAIN_SECONDS]], + [], + ['rollingUpdateDrainSeconds' => 'rolling update drain time'] + ); + + $this->application->settings->rolling_update_drain_seconds = $validated['rollingUpdateDrainSeconds']; + $changedFields = array_keys($this->application->settings->getDirty()); + $this->application->settings->save(); + $this->auditSettingsUpdate($changedFields); + + $this->dispatch('success', 'Rolling update drain time updated.'); + } catch (ValidationException $e) { + throw $e; + } catch (\Throwable $e) { + return handleError($e, $this); + } + } + public function saveMaxRestartCount() { try { diff --git a/app/Models/ApplicationSetting.php b/app/Models/ApplicationSetting.php index abaf67312e..7d2c5a849a 100644 --- a/app/Models/ApplicationSetting.php +++ b/app/Models/ApplicationSetting.php @@ -52,6 +52,7 @@ use Symfony\Component\Yaml\Yaml; 'include_source_commit_in_build' => ['type' => 'boolean'], 'docker_images_to_keep' => ['type' => 'integer'], 'stop_grace_period' => ['type' => 'integer', 'nullable' => true], + 'rolling_update_drain_seconds' => ['type' => 'integer', 'description' => 'Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off.'], ] )] class ApplicationSetting extends Model @@ -84,6 +85,7 @@ class ApplicationSetting extends Model 'is_git_shallow_clone_enabled' => 'boolean', 'docker_images_to_keep' => 'integer', 'stop_grace_period' => 'integer', + 'rolling_update_drain_seconds' => 'integer', 'is_log_drain_enabled' => 'boolean', 'is_gpu_enabled' => 'boolean', 'is_include_timestamps' => 'boolean', @@ -142,6 +144,7 @@ class ApplicationSetting extends Model 'include_source_commit_in_build', 'docker_images_to_keep', 'stop_grace_period', + 'rolling_update_drain_seconds', ]; /** @@ -168,6 +171,11 @@ class ApplicationSetting extends Model return DEFAULT_STOP_GRACE_PERIOD_SECONDS; } + public function rollingUpdateDrainSeconds(): int + { + return min(max($this->rolling_update_drain_seconds ?? DEFAULT_ROLLING_UPDATE_DRAIN_SECONDS, 0), MAX_ROLLING_UPDATE_DRAIN_SECONDS); + } + public function deploymentStopGracePeriodSeconds(): int { if (isDev() && $this->stop_grace_period === null) { diff --git a/bootstrap/helpers/api.php b/bootstrap/helpers/api.php index d6f05c8095..c4c5eb7a27 100644 --- a/bootstrap/helpers/api.php +++ b/bootstrap/helpers/api.php @@ -228,6 +228,7 @@ function sharedDataApplications(?Application $application = null): array 'preview_url_template' => 'string', 'max_restart_count' => 'integer|min:0', 'stop_grace_period' => 'nullable|integer|min:'.MIN_STOP_GRACE_PERIOD_SECONDS.'|max:'.MAX_STOP_GRACE_PERIOD_SECONDS, + 'rolling_update_drain_seconds' => 'integer|min:0|max:'.MAX_ROLLING_UPDATE_DRAIN_SECONDS, 'docker_images_to_keep' => 'integer|min:0|max:100', 'static_image' => Rule::enum(StaticImageTypes::class), 'domains' => ValidationPatterns::applicationDomainRules(), @@ -490,6 +491,7 @@ function removeUnnecessaryFieldsFromRequest(Request $request) $request->offsetUnset('include_coolify_default_variables'); $request->offsetUnset('is_pr_deployments_public_enabled'); $request->offsetUnset('stop_grace_period'); + $request->offsetUnset('rolling_update_drain_seconds'); $request->offsetUnset('docker_images_to_keep'); $request->offsetUnset('is_gzip_enabled'); $request->offsetUnset('is_stripprefix_enabled'); diff --git a/bootstrap/helpers/constants.php b/bootstrap/helpers/constants.php index 6dd1155a20..a235c9b680 100644 --- a/bootstrap/helpers/constants.php +++ b/bootstrap/helpers/constants.php @@ -63,6 +63,8 @@ const RESTART_MODE = 'unless-stopped'; const DEFAULT_STOP_GRACE_PERIOD_SECONDS = 30; const MIN_STOP_GRACE_PERIOD_SECONDS = 1; const MAX_STOP_GRACE_PERIOD_SECONDS = 3600; +const DEFAULT_ROLLING_UPDATE_DRAIN_SECONDS = 5; +const MAX_ROLLING_UPDATE_DRAIN_SECONDS = 300; const DATABASE_DOCKER_IMAGES = [ 'bitnami/mariadb', diff --git a/bootstrap/helpers/docker.php b/bootstrap/helpers/docker.php index c27eb4270d..4676b1e754 100644 --- a/bootstrap/helpers/docker.php +++ b/bootstrap/helpers/docker.php @@ -787,7 +787,7 @@ function caddySiteAddress(string $scheme, string $host, bool $isForceHttpsEnable return "{$scheme}://{$host}"; } -function fqdnLabelsForCaddy(string $network, string $uuid, Collection $domains, bool $is_force_https_enabled = false, $onlyPort = null, ?Collection $serviceLabels = null, ?bool $is_gzip_enabled = true, ?bool $is_stripprefix_enabled = true, ?string $service_name = null, ?string $image = null, string $redirect_direction = 'both', ?string $predefinedPort = null, bool $is_http_basic_auth_enabled = false, ?string $http_basic_auth_username = null, ?string $http_basic_auth_password = null, ?Collection $noindex_domains = null, bool $is_traffic_analytics_enabled = false, array $domainPortOverrides = [], bool $supports_log_append = false, bool $supports_basic_auth_directive = false, ?int $http_basic_auth_bcrypt_cost = null, array $http_basic_auth_argon2id_options = [], bool $is_permanent_redirect_enabled = false) +function fqdnLabelsForCaddy(string $network, string $uuid, Collection $domains, bool $is_force_https_enabled = false, $onlyPort = null, ?Collection $serviceLabels = null, ?bool $is_gzip_enabled = true, ?bool $is_stripprefix_enabled = true, ?string $service_name = null, ?string $image = null, string $redirect_direction = 'both', ?string $predefinedPort = null, bool $is_http_basic_auth_enabled = false, ?string $http_basic_auth_username = null, ?string $http_basic_auth_password = null, ?Collection $noindex_domains = null, bool $is_traffic_analytics_enabled = false, array $domainPortOverrides = [], bool $supports_log_append = false, bool $supports_basic_auth_directive = false, ?int $http_basic_auth_bcrypt_cost = null, array $http_basic_auth_argon2id_options = [], bool $is_permanent_redirect_enabled = false, bool $is_retry_enabled = false) { $labels = collect([]); if ($serviceLabels) { @@ -840,6 +840,11 @@ function fqdnLabelsForCaddy(string $network, string $uuid, Collection $domains, } else { $labels->push("caddy_{$loop}.{$handle}.{$loop}_reverse_proxy={{upstreams}}"); } + if ($is_retry_enabled) { + // A failed connection, for example to the old container that stops during a rolling + // update, is retried on another upstream. Caddy always retries a connection that failed. + $labels->push("caddy_{$loop}.{$handle}.{$loop}_reverse_proxy.lb_try_duration=5s"); + } $labels->push("caddy_{$loop}.{$handle}={$path}*"); if ($is_gzip_enabled) { $labels->push("caddy_{$loop}.encode=zstd gzip"); @@ -944,7 +949,7 @@ function traefikHostRulesWithoutIpv6Brackets(Collection $labels): Collection ) : $label); } -function fqdnLabelsForTraefik(string $uuid, Collection $domains, bool $is_force_https_enabled = false, $onlyPort = null, ?Collection $serviceLabels = null, ?bool $is_gzip_enabled = true, ?bool $is_stripprefix_enabled = true, ?string $service_name = null, bool $generate_unique_uuid = false, ?string $image = null, string $redirect_direction = 'both', bool $is_http_basic_auth_enabled = false, ?string $http_basic_auth_username = null, ?string $http_basic_auth_password = null, ?Collection $noindex_domains = null, bool $escape_redirect_replacement_for_compose = true, array $domainPortOverrides = [], ?string $certificate_resolver = 'letsencrypt', ?int $http_basic_auth_bcrypt_cost = null, bool $is_permanent_redirect_enabled = false) +function fqdnLabelsForTraefik(string $uuid, Collection $domains, bool $is_force_https_enabled = false, $onlyPort = null, ?Collection $serviceLabels = null, ?bool $is_gzip_enabled = true, ?bool $is_stripprefix_enabled = true, ?string $service_name = null, bool $generate_unique_uuid = false, ?string $image = null, string $redirect_direction = 'both', bool $is_http_basic_auth_enabled = false, ?string $http_basic_auth_username = null, ?string $http_basic_auth_password = null, ?Collection $noindex_domains = null, bool $escape_redirect_replacement_for_compose = true, array $domainPortOverrides = [], ?string $certificate_resolver = 'letsencrypt', ?int $http_basic_auth_bcrypt_cost = null, bool $is_permanent_redirect_enabled = false, bool $is_retry_enabled = false) { $labels = collect([]); $labels->push('traefik.enable=true'); @@ -999,6 +1004,14 @@ function fqdnLabelsForTraefik(string $uuid, Collection $domains, bool $is_force_ ->filter() ->unique(); } + if ($is_retry_enabled) { + // Sends a request to another container when the connection fails, for example to the old + // container that stops during a rolling update. Traefik does not retry after a response. + // A retry can pick the old container again under load; 3 attempts left about 5% of the + // requests failing in a live test, 5 attempts less than 1% (#8627). + $labels->push("traefik.http.middlewares.retry-{$uuid}.retry.attempts=5"); + $middlewares_from_labels->push("retry-{$uuid}"); + } foreach ($domains as $loop => $domain) { try { if ($generate_unique_uuid) { @@ -1264,6 +1277,7 @@ function generateLabelsApplication(Application $application, ?ApplicationPreview http_basic_auth_bcrypt_cost: $application->http_basic_auth_bcrypt_cost, noindex_domains: $noindexDomains, domainPortOverrides: $application->domain_port_overrides ?? [], + is_retry_enabled: true, certificate_resolver: $application->destination->server->traefikCertificateResolver($application->settings?->traefik_certificate_resolver), )); break; @@ -1288,6 +1302,7 @@ function generateLabelsApplication(Application $application, ?ApplicationPreview supports_log_append: $application->destination->server->caddySupportsLogAppend(), supports_basic_auth_directive: $application->destination->server->caddySupportsBasicAuthDirective(), domainPortOverrides: $application->domain_port_overrides ?? [], + is_retry_enabled: true, )); break; } @@ -1308,6 +1323,7 @@ function generateLabelsApplication(Application $application, ?ApplicationPreview noindex_domains: $noindexDomains, escape_redirect_replacement_for_compose: false, domainPortOverrides: $application->domain_port_overrides ?? [], + is_retry_enabled: true, certificate_resolver: $application->destination->server->traefikCertificateResolver($application->settings?->traefik_certificate_resolver), )); $labels = $labels->merge(fqdnLabelsForCaddy( @@ -1330,6 +1346,7 @@ function generateLabelsApplication(Application $application, ?ApplicationPreview supports_log_append: $application->destination->server->caddySupportsLogAppend(), supports_basic_auth_directive: $application->destination->server->caddySupportsBasicAuthDirective(), domainPortOverrides: $application->domain_port_overrides ?? [], + is_retry_enabled: true, )); } } diff --git a/database/migrations/2026_10_23_224202_add_rolling_update_drain_seconds_to_application_settings.php b/database/migrations/2026_10_23_224202_add_rolling_update_drain_seconds_to_application_settings.php new file mode 100644 index 0000000000..88edb7081f --- /dev/null +++ b/database/migrations/2026_10_23_224202_add_rolling_update_drain_seconds_to_application_settings.php @@ -0,0 +1,31 @@ +integer('rolling_update_drain_seconds') + ->default(5) + ->after('stop_grace_period') + ->comment('Seconds to wait after the new container is healthy before old containers stop during a rolling update (0-300). 0 turns the wait off.'); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('application_settings', function (Blueprint $table) { + $table->dropColumn('rolling_update_drain_seconds'); + }); + } +}; diff --git a/docs/server-transfer-full-coverage.md b/docs/server-transfer-full-coverage.md index 9aa62f2b6b..3c39e0e15d 100644 --- a/docs/server-transfer-full-coverage.md +++ b/docs/server-transfer-full-coverage.md @@ -96,7 +96,7 @@ When you add a model or a setting to the transfer, add it to this file too. | ✅ | Limits | memory, swap, swappiness, reservation, CPUs, cpuset, CPU shares | | ➕ | Advanced | custom labels, custom docker run options, pre/post deployment command + container, manual webhook secrets | | ✅ | Maintenance | maintenance mode on + custom maintenance page | -| ✅ | All application settings that are not the default | git submodules, LFS, shallow clone, auto deploy off, force HTTPS, debug, preview deployments on, PR deployments public, log drain, GPU (driver, count, device IDs, options), include timestamps, swarm-only worker nodes, build server, consistent container name, gzip off, strip prefix off, connect to Docker network, custom internal name, container label escape off, env sorting, read-only labels, preserve repository, disable build cache, use build secrets, stop grace period 30, inject build args off, include source commit, Docker images to keep 5, custom container name prefix, redact env values in logs, Coolify default variables off, Traefik certificate resolver | +| ✅ | All application settings that are not the default | git submodules, LFS, shallow clone, auto deploy off, force HTTPS, debug, preview deployments on, PR deployments public, log drain, GPU (driver, count, device IDs, options), include timestamps, swarm-only worker nodes, build server, consistent container name, gzip off, strip prefix off, connect to Docker network, custom internal name, container label escape off, env sorting, read-only labels, preserve repository, disable build cache, use build secrets, stop grace period 30, rolling update drain time 12, inject build args off, include source commit, Docker images to keep 5, custom container name prefix, redact env values in logs, Coolify default variables off, Traefik certificate resolver | | ✅ | Environment variables | normal, multiline, literal, shown once, build time only, runtime only, required, with comment, preview-only, and references to `{{team.*}}`, `{{project.*}}`, `{{environment.*}}`, `{{server.*}}` | | ✅ | Persistent volume | the seed has one; add a host-path bind mount and preview suffix on | | ✅ | File mount + directory mount | content, chown, chmod | diff --git a/openapi.json b/openapi.json index 7bd46ddba5..7af9db1b47 100644 --- a/openapi.json +++ b/openapi.json @@ -914,6 +914,12 @@ "maximum": 3600, "description": "Container stop grace period in seconds." }, + "rolling_update_drain_seconds": { + "type": "integer", + "minimum": 0, + "maximum": 300, + "description": "Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5." + }, "docker_images_to_keep": { "type": "integer", "minimum": 0, @@ -1566,6 +1572,12 @@ "maximum": 3600, "description": "Container stop grace period in seconds." }, + "rolling_update_drain_seconds": { + "type": "integer", + "minimum": 0, + "maximum": 300, + "description": "Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5." + }, "docker_images_to_keep": { "type": "integer", "minimum": 0, @@ -2218,6 +2230,12 @@ "maximum": 3600, "description": "Container stop grace period in seconds." }, + "rolling_update_drain_seconds": { + "type": "integer", + "minimum": 0, + "maximum": 300, + "description": "Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5." + }, "docker_images_to_keep": { "type": "integer", "minimum": 0, @@ -2760,6 +2778,12 @@ "maximum": 3600, "description": "Container stop grace period in seconds." }, + "rolling_update_drain_seconds": { + "type": "integer", + "minimum": 0, + "maximum": 300, + "description": "Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5." + }, "docker_images_to_keep": { "type": "integer", "minimum": 0, @@ -3282,6 +3306,12 @@ "maximum": 3600, "description": "Container stop grace period in seconds." }, + "rolling_update_drain_seconds": { + "type": "integer", + "minimum": 0, + "maximum": 300, + "description": "Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5." + }, "docker_images_to_keep": { "type": "integer", "minimum": 0, @@ -4067,6 +4097,12 @@ "maximum": 3600, "description": "Container stop grace period in seconds." }, + "rolling_update_drain_seconds": { + "type": "integer", + "minimum": 0, + "maximum": 300, + "description": "Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5." + }, "docker_images_to_keep": { "type": "integer", "minimum": 0, @@ -25104,6 +25140,10 @@ "stop_grace_period": { "type": "integer", "nullable": true + }, + "rolling_update_drain_seconds": { + "type": "integer", + "description": "Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off." } }, "type": "object" diff --git a/openapi.yaml b/openapi.yaml index 5cfd3c35c8..4018755766 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -594,6 +594,11 @@ paths: minimum: 1 maximum: 3600 description: 'Container stop grace period in seconds.' + rolling_update_drain_seconds: + type: integer + minimum: 0 + maximum: 300 + description: 'Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5.' docker_images_to_keep: type: integer minimum: 0 @@ -1027,6 +1032,11 @@ paths: minimum: 1 maximum: 3600 description: 'Container stop grace period in seconds.' + rolling_update_drain_seconds: + type: integer + minimum: 0 + maximum: 300 + description: 'Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5.' docker_images_to_keep: type: integer minimum: 0 @@ -1460,6 +1470,11 @@ paths: minimum: 1 maximum: 3600 description: 'Container stop grace period in seconds.' + rolling_update_drain_seconds: + type: integer + minimum: 0 + maximum: 300 + description: 'Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5.' docker_images_to_keep: type: integer minimum: 0 @@ -1830,6 +1845,11 @@ paths: minimum: 1 maximum: 3600 description: 'Container stop grace period in seconds.' + rolling_update_drain_seconds: + type: integer + minimum: 0 + maximum: 300 + description: 'Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5.' docker_images_to_keep: type: integer minimum: 0 @@ -2186,6 +2206,11 @@ paths: minimum: 1 maximum: 3600 description: 'Container stop grace period in seconds.' + rolling_update_drain_seconds: + type: integer + minimum: 0 + maximum: 300 + description: 'Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5.' docker_images_to_keep: type: integer minimum: 0 @@ -2709,6 +2734,11 @@ paths: minimum: 1 maximum: 3600 description: 'Container stop grace period in seconds.' + rolling_update_drain_seconds: + type: integer + minimum: 0 + maximum: 300 + description: 'Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5.' docker_images_to_keep: type: integer minimum: 0 @@ -16039,6 +16069,9 @@ components: stop_grace_period: type: integer nullable: true + rolling_update_drain_seconds: + type: integer + description: 'Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off.' type: object Environment: description: 'Environment model' diff --git a/resources/views/livewire/project/application/advanced.blade.php b/resources/views/livewire/project/application/advanced.blade.php index 8fe5f6eefa..cbf9197f86 100644 --- a/resources/views/livewire/project/application/advanced.blade.php +++ b/resources/views/livewire/project/application/advanced.blade.php @@ -126,6 +126,13 @@ helper="How long to wait for graceful shutdown during rolling updates, manual stops, and restarts. Applies to all containers for this application. Saved automatically. Default: {{ DEFAULT_STOP_GRACE_PERIOD_SECONDS }} seconds. Range: {{ MIN_STOP_GRACE_PERIOD_SECONDS }}-{{ MAX_STOP_GRACE_PERIOD_SECONDS }} seconds (1 hour)." min="{{ MIN_STOP_GRACE_PERIOD_SECONDS }}" max="{{ MAX_STOP_GRACE_PERIOD_SECONDS }}" canGate="update" :canResource="$application" /> + @if ($application->build_pack !== 'dockercompose') + + @endif true, 'is_pr_deployments_public_enabled' => true, 'stop_grace_period' => 45, + 'rolling_update_drain_seconds' => 12, 'docker_images_to_keep' => 7, 'is_gzip_enabled' => false, 'is_stripprefix_enabled' => false, @@ -345,6 +346,18 @@ test('validates stop grace period bounds', function (int $stopGracePeriod) { 'above maximum' => 3601, ]); +test('validates rolling update drain time bounds', function (int $seconds) { + $this->withHeaders(applicationSettingsApiHeaders($this->bearerToken)) + ->patchJson("/api/v1/applications/{$this->application->uuid}", [ + 'rolling_update_drain_seconds' => $seconds, + ]) + ->assertUnprocessable() + ->assertJsonValidationErrors('rolling_update_drain_seconds'); +})->with([ + 'below minimum' => -1, + 'above maximum' => 301, +]); + test('validates Docker image retention bounds', function (int $dockerImagesToKeep) { $this->withHeaders(applicationSettingsApiHeaders($this->bearerToken)) ->patchJson("/api/v1/applications/{$this->application->uuid}", [ diff --git a/tests/Feature/RollingUpdateProxyDrainTest.php b/tests/Feature/RollingUpdateProxyDrainTest.php new file mode 100644 index 0000000000..2c95ddf943 --- /dev/null +++ b/tests/Feature/RollingUpdateProxyDrainTest.php @@ -0,0 +1,214 @@ + InstanceSettings::firstOrCreate(['id' => 0])); + config(['constants.ssh.mux_enabled' => false]); +}); + +class TestableRollingUpdateDrainDeploymentJob extends ApplicationDeploymentJob +{ + public function __construct() {} +} + +function createApplicationForRollingUpdateDrainTest(string $proxyType = 'TRAEFIK', bool $exactLabels = false): Application +{ + $team = Team::factory()->create(); + $project = Project::factory()->create(['team_id' => $team->id]); + $environment = Environment::factory()->create(['project_id' => $project->id]); + $server = Server::factory()->create([ + 'team_id' => $team->id, + 'private_key_id' => PrivateKey::factory()->create(['team_id' => $team->id])->id, + 'proxy' => ['type' => $proxyType], + ]); + $server->settings->update(['generate_exact_labels' => $exactLabels]); + $destination = StandaloneDocker::query()->where('server_id', $server->id)->firstOrFail(); + + return Application::factory()->createOne([ + 'environment_id' => $environment->id, + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + 'fqdn' => 'https://example.com', + 'ports_exposes' => '3000', + ])->fresh(); +} + +/** + * Runs the drain step of a rolling update against a faked server and returns the deployment log lines. + * + * @param array|null $oldProxyLabels labels of the running old container, null when there is none + * @param array $newProxyLabels + * @return list + */ +function runRollingUpdateDrainStep(Application $application, bool $newVersionIsHealthy, ?array $oldProxyLabels = ['traefik.enable' => 'true'], array $newProxyLabels = ['traefik.enable' => 'true']): array +{ + Process::fake(function ($process) use ($application, $oldProxyLabels, $newProxyLabels) { + $command = is_array($process->command) ? implode(' ', $process->command) : $process->command; + if (str_contains($command, 'docker inspect')) { + return Process::result(output: json_encode(['coolify.name' => 'new', ...$newProxyLabels])."\n".json_encode(['coolify.name' => 'old', ...$oldProxyLabels])); + } + if (str_contains($command, 'docker ps') && str_contains($command, "coolify.applicationUuid={$application->uuid}")) { + $containers = [['ID' => 'new-id', 'Names' => 'app-new', 'State' => 'running', 'Labels' => '']]; + if ($oldProxyLabels !== null) { + $containers[] = ['ID' => 'old-id', 'Names' => 'app-old', 'State' => 'running', 'Labels' => '']; + } + + return Process::result(output: collect($containers)->map(fn (array $container) => json_encode($container))->implode("\n")); + } + + return Process::result(output: ''); + }); + + $logs = []; + $queue = Mockery::mock(ApplicationDeploymentQueue::class)->makePartial(); + $queue->shouldReceive('addLogEntry')->andReturnUsing(function (string $message) use (&$logs) { + $logs[] = $message; + }); + + $job = new TestableRollingUpdateDrainDeploymentJob; + $reflection = new ReflectionClass(ApplicationDeploymentJob::class); + foreach ([ + 'application' => $application->fresh(), + 'application_deployment_queue' => $queue, + 'deployment_uuid' => 'rolling-update-drain', + 'server' => $application->destination->server, + 'container_name' => 'app-new', + 'pull_request_id' => 0, + ] as $property => $value) { + $reflection->getProperty($property)->setValue($job, $value); + } + + Cache::forget('deployment-new-version-healthy:rolling-update-drain'); + if ($newVersionIsHealthy) { + Cache::put('deployment-new-version-healthy:rolling-update-drain', true); + } + + $reflection->getMethod('wait_for_proxy_to_route_new_container')->invoke($job); + + return $logs; +} + +it('retries failed connections on another container in Traefik application labels', function (bool $exactLabels) { + $application = createApplicationForRollingUpdateDrainTest(ProxyTypes::TRAEFIK->value, $exactLabels); + + $labels = collect(generateLabelsApplication($application)); + + expect($labels)->toContain("traefik.http.middlewares.retry-{$application->uuid}.retry.attempts=5"); + $routerMiddlewares = $labels->filter(fn (string $label) => str_contains($label, "routers.https-0-{$application->uuid}.middlewares=")); + expect($routerMiddlewares)->toHaveCount(1) + ->and($routerMiddlewares->first())->toContain("retry-{$application->uuid}"); +})->with([ + 'generic labels' => false, + 'exact labels' => true, +]); + +it('retries failed connections on another upstream in Caddy application labels', function () { + $application = createApplicationForRollingUpdateDrainTest(ProxyTypes::CADDY->value, exactLabels: true); + + expect(generateLabelsApplication($application)) + ->toContain('caddy_0.handle_path.0_reverse_proxy.lb_try_duration=5s'); +}); + +it('does not add retry labels to preview deployments', function () { + $application = createApplicationForRollingUpdateDrainTest(); + $preview = ApplicationPreview::forceCreate([ + 'application_id' => $application->id, + 'pull_request_id' => 7, + 'pull_request_html_url' => 'https://github.com/coollabsio/coolify/pull/7', + 'fqdn' => 'https://pr-7.example.com', + ]); + + expect(implode("\n", generateLabelsApplication($application, $preview))) + ->not->toContain('retry') + ->not->toContain('lb_try_duration'); +}); + +it('waits for the proxy before old containers stop when the new container is healthy', function () { + Sleep::fake(); + $application = createApplicationForRollingUpdateDrainTest(); + + $logs = runRollingUpdateDrainStep($application, newVersionIsHealthy: true); + + Sleep::assertSequence([Sleep::for(DEFAULT_ROLLING_UPDATE_DRAIN_SECONDS)->seconds()]); + expect($logs)->toBe(['Waiting 5 seconds for the proxy to send traffic to the new container.']); +}); + +it('does not wait when the drain time is 0 or the new container is not healthy', function (int $drainSeconds, bool $newVersionIsHealthy) { + Sleep::fake(); + $application = createApplicationForRollingUpdateDrainTest(); + $application->settings->update(['rolling_update_drain_seconds' => $drainSeconds]); + + $logs = runRollingUpdateDrainStep($application, $newVersionIsHealthy); + + Sleep::assertNeverSlept(); + expect($logs)->toBe([]); +})->with([ + 'drain turned off' => [0, true], + 'new container not healthy' => [5, false], +]); + +it('does not wait when no old container runs', function () { + Sleep::fake(); + $application = createApplicationForRollingUpdateDrainTest(); + + $logs = runRollingUpdateDrainStep($application, newVersionIsHealthy: true, oldProxyLabels: null); + + Sleep::assertNeverSlept(); + expect($logs)->toBe([]); +}); + +it('stops the old containers without waiting when the proxy labels changed', function () { + Sleep::fake(); + $application = createApplicationForRollingUpdateDrainTest(); + + $logs = runRollingUpdateDrainStep( + $application, + newVersionIsHealthy: true, + oldProxyLabels: ['traefik.http.routers.http-0-app.middlewares' => 'gzip-sse'], + newProxyLabels: ['traefik.http.routers.http-0-app.middlewares' => 'gzip-sse,retry-app'], + ); + + Sleep::assertNeverSlept(); + expect($logs)->toBe(['The proxy labels changed, so the old containers stop without waiting.']); +}); + +it('saves the rolling update drain time on the advanced page', function () { + $user = User::factory()->create(); + $application = createApplicationForRollingUpdateDrainTest(); + $application->team()->members()->attach($user->id, ['role' => 'owner']); + $this->actingAs($user); + session(['currentTeam' => $application->team()]); + + Livewire::test(Advanced::class, ['application' => $application]) + ->assertSet('rollingUpdateDrainSeconds', DEFAULT_ROLLING_UPDATE_DRAIN_SECONDS) + ->set('rollingUpdateDrainSeconds', 20) + ->call('saveRollingUpdateDrainSeconds') + ->assertHasNoErrors() + ->set('rollingUpdateDrainSeconds', 301) + ->call('saveRollingUpdateDrainSeconds') + ->assertHasErrors(['rollingUpdateDrainSeconds' => ['max']]); + + expect($application->settings->fresh()->rolling_update_drain_seconds)->toBe(20); +});