From 5dacaed9f27d19f722a1b9769a3aae33bc11e4c6 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 28 Oct 2026 12:10:27 +0100 Subject: [PATCH] fix(proxy): keep the application proxy labels of v4.5.0 The retry labels (Traefik retry middleware, Caddy lb_try_duration) changed the labels of every application. On the first rolling update after a settings save, the old and the new container defined the same Traefik router differently, so Traefik dropped the route and returned 503 until the old container stopped. Remove the retry labels from this release. The rolling update drain wait stays. --- bootstrap/helpers/docker.php | 21 ++-------------- tests/Feature/RollingUpdateProxyDrainTest.php | 25 ++++++------------- 2 files changed, 10 insertions(+), 36 deletions(-) diff --git a/bootstrap/helpers/docker.php b/bootstrap/helpers/docker.php index f179df1d96..d3ca519766 100644 --- a/bootstrap/helpers/docker.php +++ b/bootstrap/helpers/docker.php @@ -801,7 +801,7 @@ function caddySiteAddress(string $scheme, string $host, bool $isForceHttpsEnable return "{$scheme}://{$host}"; } -function fqdnLabelsForCaddy(string $network, string $uuid, Collection $domains, bool $is_force_https_enabled = false, $onlyPort = null, ?Collection $serviceLabels = null, ?bool $is_gzip_enabled = true, ?bool $is_stripprefix_enabled = true, ?string $service_name = null, ?string $image = null, string $redirect_direction = 'both', ?string $predefinedPort = null, bool $is_http_basic_auth_enabled = false, ?string $http_basic_auth_username = null, ?string $http_basic_auth_password = null, ?Collection $noindex_domains = null, bool $is_traffic_analytics_enabled = false, array $domainPortOverrides = [], bool $supports_log_append = false, bool $supports_basic_auth_directive = false, ?int $http_basic_auth_bcrypt_cost = null, array $http_basic_auth_argon2id_options = [], bool $is_permanent_redirect_enabled = false, bool $is_retry_enabled = false) +function fqdnLabelsForCaddy(string $network, string $uuid, Collection $domains, bool $is_force_https_enabled = false, $onlyPort = null, ?Collection $serviceLabels = null, ?bool $is_gzip_enabled = true, ?bool $is_stripprefix_enabled = true, ?string $service_name = null, ?string $image = null, string $redirect_direction = 'both', ?string $predefinedPort = null, bool $is_http_basic_auth_enabled = false, ?string $http_basic_auth_username = null, ?string $http_basic_auth_password = null, ?Collection $noindex_domains = null, bool $is_traffic_analytics_enabled = false, array $domainPortOverrides = [], bool $supports_log_append = false, bool $supports_basic_auth_directive = false, ?int $http_basic_auth_bcrypt_cost = null, array $http_basic_auth_argon2id_options = [], bool $is_permanent_redirect_enabled = false) { $labels = collect([]); if ($serviceLabels) { @@ -854,11 +854,6 @@ function fqdnLabelsForCaddy(string $network, string $uuid, Collection $domains, } else { $labels->push("caddy_{$loop}.{$handle}.{$loop}_reverse_proxy={{upstreams}}"); } - if ($is_retry_enabled) { - // A failed connection, for example to the old container that stops during a rolling - // update, is retried on another upstream. Caddy always retries a connection that failed. - $labels->push("caddy_{$loop}.{$handle}.{$loop}_reverse_proxy.lb_try_duration=5s"); - } $labels->push("caddy_{$loop}.{$handle}={$path}*"); if ($is_gzip_enabled) { $labels->push("caddy_{$loop}.encode=zstd gzip"); @@ -963,7 +958,7 @@ function traefikHostRulesWithoutIpv6Brackets(Collection $labels): Collection ) : $label); } -function fqdnLabelsForTraefik(string $uuid, Collection $domains, bool $is_force_https_enabled = false, $onlyPort = null, ?Collection $serviceLabels = null, ?bool $is_gzip_enabled = true, ?bool $is_stripprefix_enabled = true, ?string $service_name = null, bool $generate_unique_uuid = false, ?string $image = null, string $redirect_direction = 'both', bool $is_http_basic_auth_enabled = false, ?string $http_basic_auth_username = null, ?string $http_basic_auth_password = null, ?Collection $noindex_domains = null, bool $escape_redirect_replacement_for_compose = true, array $domainPortOverrides = [], ?string $certificate_resolver = 'letsencrypt', ?int $http_basic_auth_bcrypt_cost = null, bool $is_permanent_redirect_enabled = false, bool $is_retry_enabled = false) +function fqdnLabelsForTraefik(string $uuid, Collection $domains, bool $is_force_https_enabled = false, $onlyPort = null, ?Collection $serviceLabels = null, ?bool $is_gzip_enabled = true, ?bool $is_stripprefix_enabled = true, ?string $service_name = null, bool $generate_unique_uuid = false, ?string $image = null, string $redirect_direction = 'both', bool $is_http_basic_auth_enabled = false, ?string $http_basic_auth_username = null, ?string $http_basic_auth_password = null, ?Collection $noindex_domains = null, bool $escape_redirect_replacement_for_compose = true, array $domainPortOverrides = [], ?string $certificate_resolver = 'letsencrypt', ?int $http_basic_auth_bcrypt_cost = null, bool $is_permanent_redirect_enabled = false) { $labels = collect([]); $labels->push('traefik.enable=true'); @@ -1018,14 +1013,6 @@ function fqdnLabelsForTraefik(string $uuid, Collection $domains, bool $is_force_ ->filter() ->unique(); } - if ($is_retry_enabled) { - // Sends a request to another container when the connection fails, for example to the old - // container that stops during a rolling update. Traefik does not retry after a response. - // A retry can pick the old container again under load; 3 attempts left about 5% of the - // requests failing in a live test, 5 attempts less than 1% (#8627). - $labels->push("traefik.http.middlewares.retry-{$uuid}.retry.attempts=5"); - $middlewares_from_labels->push("retry-{$uuid}"); - } foreach ($domains as $loop => $domain) { try { if ($generate_unique_uuid) { @@ -1291,7 +1278,6 @@ function generateLabelsApplication(Application $application, ?ApplicationPreview http_basic_auth_bcrypt_cost: $application->http_basic_auth_bcrypt_cost, noindex_domains: $noindexDomains, domainPortOverrides: $application->domain_port_overrides ?? [], - is_retry_enabled: true, certificate_resolver: $application->destination->server->traefikCertificateResolver($application->settings?->traefik_certificate_resolver), )); break; @@ -1316,7 +1302,6 @@ function generateLabelsApplication(Application $application, ?ApplicationPreview supports_log_append: $application->destination->server->caddySupportsLogAppend(), supports_basic_auth_directive: $application->destination->server->caddySupportsBasicAuthDirective(), domainPortOverrides: $application->domain_port_overrides ?? [], - is_retry_enabled: true, )); break; } @@ -1337,7 +1322,6 @@ function generateLabelsApplication(Application $application, ?ApplicationPreview noindex_domains: $noindexDomains, escape_redirect_replacement_for_compose: false, domainPortOverrides: $application->domain_port_overrides ?? [], - is_retry_enabled: true, certificate_resolver: $application->destination->server->traefikCertificateResolver($application->settings?->traefik_certificate_resolver), )); $labels = $labels->merge(fqdnLabelsForCaddy( @@ -1360,7 +1344,6 @@ function generateLabelsApplication(Application $application, ?ApplicationPreview supports_log_append: $application->destination->server->caddySupportsLogAppend(), supports_basic_auth_directive: $application->destination->server->caddySupportsBasicAuthDirective(), domainPortOverrides: $application->domain_port_overrides ?? [], - is_retry_enabled: true, )); } } diff --git a/tests/Feature/RollingUpdateProxyDrainTest.php b/tests/Feature/RollingUpdateProxyDrainTest.php index 4ad85d0d63..2ad2e355d3 100644 --- a/tests/Feature/RollingUpdateProxyDrainTest.php +++ b/tests/Feature/RollingUpdateProxyDrainTest.php @@ -110,27 +110,18 @@ function runRollingUpdateDrainStep(Application $application, bool $newVersionIsH return $logs; } -it('retries failed connections on another container in Traefik application labels', function (bool $exactLabels) { - $application = createApplicationForRollingUpdateDrainTest(ProxyTypes::TRAEFIK->value, $exactLabels); +it('keeps the application proxy labels of v4.5.0 without a retry, so a redeploy after an upgrade does not change them', function (string $proxyType, bool $exactLabels) { + $application = createApplicationForRollingUpdateDrainTest($proxyType, $exactLabels); - $labels = collect(generateLabelsApplication($application)); - - expect($labels)->toContain("traefik.http.middlewares.retry-{$application->uuid}.retry.attempts=5"); - $routerMiddlewares = $labels->filter(fn (string $label) => str_contains($label, "routers.https-0-{$application->uuid}.middlewares=")); - expect($routerMiddlewares)->toHaveCount(1) - ->and($routerMiddlewares->first())->toContain("retry-{$application->uuid}"); + expect(implode("\n", generateLabelsApplication($application))) + ->not->toContain('retry') + ->not->toContain('lb_try_duration'); })->with([ - 'generic labels' => false, - 'exact labels' => true, + 'Traefik generic labels' => [ProxyTypes::TRAEFIK->value, false], + 'Traefik exact labels' => [ProxyTypes::TRAEFIK->value, true], + 'Caddy exact labels' => [ProxyTypes::CADDY->value, true], ]); -it('retries failed connections on another upstream in Caddy application labels', function () { - $application = createApplicationForRollingUpdateDrainTest(ProxyTypes::CADDY->value, exactLabels: true); - - expect(generateLabelsApplication($application)) - ->toContain('caddy_0.handle_path.0_reverse_proxy.lb_try_duration=5s'); -}); - it('does not add retry labels to preview deployments', function () { $application = createApplicationForRollingUpdateDrainTest(); $preview = ApplicationPreview::forceCreate([