From a2c048c9999e7c0bf3266c1724f42b8e6e66e2e7 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Tue, 27 Oct 2026 16:24:55 +0100 Subject: [PATCH] fix(database): recreate TCP proxies that run an outdated configuration Proxies started before v4.4.4 kept the old nginx config after an upgrade: the upstream IP was resolved only once and IPv6 connections were reset. - Label new proxies with a configuration version - Status checks (Sentinel and SSH) recreate a running proxy with an older or missing version once, at most every 10 minutes - Remove the old proxy only after the image pull, so a failed pull keeps it running Closes #10198 Co-Authored-By: Claude Opus 5.5 --- app/Actions/Database/StartDatabaseProxy.php | 28 +++- app/Actions/Docker/GetContainersStatus.php | 4 + app/Jobs/PushServerUpdateJob.php | 2 + .../DatabaseProxyConfigRefreshTest.php | 147 ++++++++++++++++++ 4 files changed, 180 insertions(+), 1 deletion(-) create mode 100644 tests/Feature/DatabaseProxyConfigRefreshTest.php diff --git a/app/Actions/Database/StartDatabaseProxy.php b/app/Actions/Database/StartDatabaseProxy.php index 3070e88ac6..7c954d9501 100644 --- a/app/Actions/Database/StartDatabaseProxy.php +++ b/app/Actions/Database/StartDatabaseProxy.php @@ -13,6 +13,7 @@ use App\Models\StandaloneMysql; use App\Models\StandalonePostgresql; use App\Models\StandaloneRedis; use App\Notifications\Container\ContainerRestarted; +use Illuminate\Support\Facades\Cache; use Lorisleiva\Actions\Concerns\AsAction; use Lorisleiva\Actions\Decorators\JobDecorator; use Symfony\Component\Yaml\Yaml; @@ -21,6 +22,27 @@ class StartDatabaseProxy { use AsAction; + /** + * Version of the generated proxy configuration. Raise it when the configuration changes, so the + * status checks recreate proxies that an older Coolify version started with the old configuration. + */ + public const CONFIG_VERSION = '2'; + + public const CONFIG_VERSION_LABEL = 'coolify.databaseProxyConfigVersion'; + + /** + * True when a running proxy has an older configuration and no recreation was claimed in the last + * ten minutes. The claim stops a busy queue from recreating the proxy again before the first run. + */ + public static function claimOutdatedRefresh(string $databaseUuid, mixed $labels): bool + { + if ((string) collect($labels)->get(self::CONFIG_VERSION_LABEL) === self::CONFIG_VERSION) { + return false; + } + + return Cache::add("database-proxy-refresh:{$databaseUuid}", true, now()->addMinutes(10)); + } + public function configureJob(JobDecorator $job): void { $job->onQueue(deployment_queue()); @@ -85,6 +107,9 @@ class StartDatabaseProxy 'image' => 'nginx:stable-alpine', 'container_name' => $proxyContainerName, 'restart' => RESTART_MODE, + 'labels' => [ + self::CONFIG_VERSION_LABEL => self::CONFIG_VERSION, + ], 'ports' => [ "$database->public_port:$database->public_port", ], @@ -120,7 +145,6 @@ class StartDatabaseProxy ]; $dockercompose_base64 = base64_encode(Yaml::dump($docker_compose, 4, 2)); $nginxconf_base64 = base64_encode($nginxconf); - instant_remote_process(["docker rm -f $proxyContainerName"], $server, false); try { instant_remote_process([ @@ -129,6 +153,8 @@ class StartDatabaseProxy "echo '{$dockercompose_base64}' | base64 -d | tee $configuration_dir/docker-compose.yaml > /dev/null", // Pass -f so Compose uses only the file that Coolify wrote. "docker compose --project-directory {$configuration_dir} -f {$configuration_dir}/docker-compose.yaml pull", + // Remove the old proxy only after the pull, so a failed pull keeps it running. + dockerRemoveCommand($proxyContainerName), "docker compose --project-directory {$configuration_dir} -f {$configuration_dir}/docker-compose.yaml up -d", ], $server); } catch (\RuntimeException $e) { diff --git a/app/Actions/Docker/GetContainersStatus.php b/app/Actions/Docker/GetContainersStatus.php index 7faeb37b78..c59545a9dd 100644 --- a/app/Actions/Docker/GetContainersStatus.php +++ b/app/Actions/Docker/GetContainersStatus.php @@ -241,6 +241,8 @@ class GetContainersStatus implements ShouldBeUnique })->first(); if (! $foundTcpProxy) { StartDatabaseProxy::run($service_db); + } elseif (! $this->server->isSwarm() && StartDatabaseProxy::claimOutdatedRefresh($proxyUuid, data_get($foundTcpProxy, 'Config.Labels'))) { + StartDatabaseProxy::run($service_db); } } else { // Clean up orphaned proxy when is_public=false @@ -292,6 +294,8 @@ class GetContainersStatus implements ShouldBeUnique })->first(); if (! $foundTcpProxy) { StartDatabaseProxy::run($database); + } elseif (! $this->server->isSwarm() && StartDatabaseProxy::claimOutdatedRefresh($uuid, data_get($foundTcpProxy, 'Config.Labels'))) { + StartDatabaseProxy::run($database); } } else { // Clean up orphaned proxy when is_public=false diff --git a/app/Jobs/PushServerUpdateJob.php b/app/Jobs/PushServerUpdateJob.php index 92df6bd8c6..07c069c15d 100644 --- a/app/Jobs/PushServerUpdateJob.php +++ b/app/Jobs/PushServerUpdateJob.php @@ -864,6 +864,8 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced if (! $tcpProxyContainerFound) { StartDatabaseProxy::dispatch($database); $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$database->name}", $this->server, restartedResource: $database)); + } elseif (StartDatabaseProxy::claimOutdatedRefresh($databaseUuid, data_get($tcpProxyContainerFound, 'labels'))) { + StartDatabaseProxy::dispatch($database); } } elseif ($this->isRunning($containerStatus) && ! $tcpProxy) { // Clean up orphaned proxy containers when is_public=false diff --git a/tests/Feature/DatabaseProxyConfigRefreshTest.php b/tests/Feature/DatabaseProxyConfigRefreshTest.php new file mode 100644 index 0000000000..4705f73fd3 --- /dev/null +++ b/tests/Feature/DatabaseProxyConfigRefreshTest.php @@ -0,0 +1,147 @@ +team = Team::factory()->create(); + $this->server = Server::factory()->create([ + 'team_id' => $this->team->id, + 'private_key_id' => PrivateKey::factory()->create(['team_id' => $this->team->id])->id, + ]); + $this->server->settings()->update(['is_reachable' => true, 'is_usable' => true]); + $project = Project::factory()->create(['team_id' => $this->team->id]); + $environment = Environment::factory()->create(['project_id' => $project->id]); + $destination = $this->server->standaloneDockers()->firstOrFail(); + $this->database = StandalonePostgresql::create([ + 'name' => 'public-db', + 'postgres_user' => 'postgres', + 'postgres_password' => encrypt('password'), + 'postgres_db' => 'app', + 'image' => 'postgres:16-alpine', + 'is_public' => true, + 'public_port' => 54321, + 'environment_id' => $environment->id, + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + ]); +}); + +function sentinelPushWithProxy(Server $server, StandalonePostgresql $database, array $proxyLabels): void +{ + (new PushServerUpdateJob($server, [ + 'containers' => [ + [ + 'name' => $database->uuid, + 'state' => 'running', + 'health_status' => 'healthy', + 'labels' => ['coolify.managed' => true, 'com.docker.compose.service' => $database->uuid], + ], + [ + 'name' => "{$database->uuid}-proxy", + 'state' => 'running', + 'health_status' => 'healthy', + 'labels' => ['com.docker.compose.service' => "{$database->uuid}-proxy", ...$proxyLabels], + ], + ], + 'filesystem_usage_root' => ['used_percentage' => 10], + ]))->handle(); +} + +function statusCheckWithProxy(Server $server, StandalonePostgresql $database, array $proxyLabels): void +{ + GetContainersStatus::run($server->fresh(), collect([ + [ + 'Name' => '/'.$database->uuid, + 'State' => ['Status' => 'running', 'Health' => ['Status' => 'healthy']], + 'RestartCount' => 0, + 'Config' => ['Labels' => [ + 'coolify.managed' => 'true', + 'coolify.type' => 'database', + 'com.docker.compose.service' => $database->uuid, + ]], + ], + [ + 'Name' => "/{$database->uuid}-proxy", + 'State' => ['Status' => 'running', 'Health' => ['Status' => 'healthy']], + 'RestartCount' => 0, + 'Config' => ['Labels' => ['com.docker.compose.service' => "{$database->uuid}-proxy", ...$proxyLabels]], + ], + ]), collect()); +} + +it('recreates a running TCP proxy that an older Coolify version started, once', function () { + sentinelPushWithProxy($this->server, $this->database, []); + sentinelPushWithProxy($this->server, $this->database, []); + + StartDatabaseProxy::assertPushed(1); + Notification::assertNotSentTo($this->team, ContainerRestarted::class); +}); + +it('keeps a running TCP proxy that has the current configuration', function () { + sentinelPushWithProxy($this->server, $this->database, [ + StartDatabaseProxy::CONFIG_VERSION_LABEL => StartDatabaseProxy::CONFIG_VERSION, + ]); + + StartDatabaseProxy::assertNotPushed(); +}); + +it('recreates an outdated TCP proxy in the SSH status check', function () { + StartDatabaseProxy::shouldRun()->once(); + + statusCheckWithProxy($this->server, $this->database, []); + statusCheckWithProxy($this->server, $this->database, []); +}); + +it('keeps a current TCP proxy in the SSH status check', function () { + StartDatabaseProxy::shouldRun()->never(); + + statusCheckWithProxy($this->server, $this->database, [ + StartDatabaseProxy::CONFIG_VERSION_LABEL => StartDatabaseProxy::CONFIG_VERSION, + ]); +}); + +it('labels a new TCP proxy with the current configuration and removes the old one only after the pull', function () { + $scripts = []; + Process::fake(function ($process) use (&$scripts) { + $script = remoteProcessScript($process); + $scripts[] = is_array($script) ? implode("\n", $script) : $script; + + return Process::result(output: ''); + }); + + StartDatabaseProxy::run($this->database->fresh()); + + $script = collect($scripts)->first(fn (string $script) => str_contains($script, 'docker-compose.yaml pull')); + preg_match("/echo '([A-Za-z0-9+\/=]+)' \| base64 -d \| tee [^ ]+docker-compose\.yaml/", $script, $matches); + $compose = Yaml::parse(base64_decode($matches[1])); + $pull = strpos($script, 'docker-compose.yaml pull'); + $remove = strpos($script, dockerRemoveCommand("{$this->database->uuid}-proxy")); + $up = strpos($script, 'docker-compose.yaml up -d'); + + expect($compose['services']["{$this->database->uuid}-proxy"]['labels']) + ->toBe([StartDatabaseProxy::CONFIG_VERSION_LABEL => StartDatabaseProxy::CONFIG_VERSION]) + ->and($remove)->toBeGreaterThan($pull) + ->and($up)->toBeGreaterThan($remove); +});