coolify/app/Actions/Docker/GetContainersStatus.php

<?php

namespace App\Actions\Docker;

use App\Actions\Application\StopApplication;
use App\Actions\Application\StopApplicationPreview;
use App\Actions\Database\StartDatabaseProxy;
use App\Actions\Database\StopDatabaseProxy;
use App\Actions\Service\StopServiceApplication;
use App\Actions\Shared\ComplexStatusCheck;
use App\Events\ServiceChecked;
use App\Jobs\NotifyResourceStoppedJob;
use App\Models\Application;
use App\Models\ApplicationPreview;
use App\Models\Server;
use App\Models\ServiceDatabase;
use App\Notifications\Application\RestartLimitReached as ApplicationRestartLimitReached;
use App\Services\ContainerStatusAggregator;
use App\Services\RestartCountTracker;
use App\Support\Actions\UniqueUntilProcessingJobDecorator;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Support\Arr;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\DB;
use Lorisleiva\Actions\Concerns\AsAction;
use Lorisleiva\Actions\Decorators\UniqueJobDecorator;

/**
 * Queued runs are coalesced per server: while one run waits in the queue, further dispatches are
 * dropped because the waiting run reads the same container snapshot. The lock is released when a
 * worker starts the run, so a request made during a run still queues one follow-up.
 * Synchronous run() calls are not affected.
 */
class GetContainersStatus implements ShouldBeUnique
{
    use AsAction;

    public string $jobQueue = 'high';

    public int $jobTimeout = 120;

    /**
     * Bounds how long a lost queued run (for example a killed worker) blocks new refreshes.
     */
    public int $jobUniqueFor = 60;

    public $applications;

    public ?Collection $containers;

    public ?Collection $containerReplicates;

    public $server;

    protected ?Collection $applicationContainerStatuses;

    protected ?Collection $applicationContainerRestartCounts;

    protected ?Collection $previewContainerRestartCounts;

    protected ?Collection $serviceContainerStatuses;

    protected ?Collection $serviceContainerRestartCounts;

    public static function makeUniqueJob(mixed ...$arguments): UniqueJobDecorator
    {
        return new UniqueUntilProcessingJobDecorator(static::class, ...$arguments);
    }

    public function getJobUniqueId(Server $server): string
    {
        return $server->uuid;
    }

    public function handle(Server $server, ?Collection $containers = null, ?Collection $containerReplicates = null)
    {
        $this->containers = $containers;
        $this->containerReplicates = $containerReplicates;
        $this->server = $server;
        if (! $this->server->isFunctional()) {
            return 'Server is not functional.';
        }
        $this->applications = $this->server->applications();
        $skip_these_applications = collect([]);
        foreach ($this->applications as $application) {
            if ($application->additional_servers->count() > 0) {
                $skip_these_applications->push($application);
                ComplexStatusCheck::run($application);
                $this->applications = $this->applications->filter(function ($value, $key) use ($application) {
                    return $value->id !== $application->id;
                });
            }
        }
        $this->applications = $this->applications->filter(function ($value, $key) use ($skip_these_applications) {
            return ! $skip_these_applications->pluck('id')->contains($value->id);
        });
        if ($this->containers === null) {
            ['containers' => $this->containers, 'containerReplicates' => $this->containerReplicates] = $this->server->getContainers();
        }

        if (is_null($this->containers)) {
            return;
        }

        if ($this->containerReplicates) {
            foreach ($this->containerReplicates as $containerReplica) {
                $name = data_get($containerReplica, 'Name');
                $this->containers = $this->containers->map(function ($container) use ($name, $containerReplica) {
                    if (data_get($container, 'Spec.Name') === $name) {
                        $replicas = data_get($containerReplica, 'Replicas');
                        $running = str($replicas)->explode('/')[0];
                        $total = str($replicas)->explode('/')[1];
                        if ($running === $total) {
                            data_set($container, 'State.Status', 'running');
                            data_set($container, 'State.Health.Status', 'healthy');
                        } else {
                            data_set($container, 'State.Status', 'starting');
                            data_set($container, 'State.Health.Status', 'unhealthy');
                        }
                    }

                    return $container;
                });
            }
        }
        $databases = $this->server->databases();
        $services = $this->server->services()->get();
        $previews = $this->server->previews();
        $upResourceIds = NotifyResourceStoppedJob::upResourceIds(collect($this->applications)
            ->concat($previews)
            ->concat($databases)
            ->concat($services->flatMap(fn ($service) => $service->applications))
            ->concat($services->flatMap(fn ($service) => $service->databases)));
        $foundApplications = [];
        $foundApplicationPreviews = [];
        $foundDatabases = [];
        $foundServices = [];

        // Owners of preview containers outside this server's applications, in one query.
        $foreignApplicationIdsByUuid = containerApplicationIdsByUuid(
            $this->applications,
            $this->containers
                ->map(fn ($container) => format_docker_labels_to_json(data_get($container, $this->server->isSwarm() ? 'Spec.Labels' : 'Config.Labels') ?? []))
                ->filter(fn (Collection $labels) => (bool) $labels->get('coolify.pullRequestId') && isContainerOfType($labels, 'application'))
        );

        foreach ($this->containers as $container) {
            if ($this->server->isSwarm()) {
                $labels = data_get($container, 'Spec.Labels');
                $uuid = data_get($labels, 'coolify.name');
            } else {
                $labels = data_get($container, 'Config.Labels');
            }
            $containerStatus = ContainerStatusAggregator::containerStatus($container);
            $flatLabels = format_docker_labels_to_json($labels);
            $labels = Arr::undot($flatLabels);
            if (filter_var(data_get($labels, 'com.docker.compose.oneoff'), FILTER_VALIDATE_BOOLEAN)) {
                continue;
            }
            // Containers are matched by owner UUID; numeric ids differ between instances.
            $isApplicationContainer = isContainerOfType($flatLabels, 'application');
            $isServiceContainer = isContainerOfType($flatLabels, 'service');
            [$parentService, $serviceSubType, $serviceSubResource] = resolveServiceContainerOwner($services, $flatLabels);
            if ($isApplicationContainer) {
                $pullRequestId = data_get($labels, 'coolify.pullRequestId');
                if ($pullRequestId) {
                    $applicationId = resolveContainerApplicationId($this->applications, $flatLabels, $foreignApplicationIdsByUuid);
                    $preview = $applicationId
                        ? ApplicationPreview::where('application_id', $applicationId)->where('pull_request_id', $pullRequestId)->first()
                        : null;
                    if ($preview) {
                        $foundApplicationPreviews[] = $preview->id;
                        $statusFromDb = $preview->status;
                        if ($statusFromDb !== $containerStatus) {
                            $preview->update(['status' => $containerStatus]);
                        } else {
                            $preview->update(['last_online_at' => now()]);
                        }
                        $key = $applicationId.':'.$pullRequestId;
                        $this->previewContainerRestartCounts ??= collect();
                        $this->previewContainerRestartCounts->push([
                            'key' => $key,
                            'count' => (int) data_get($container, 'RestartCount', 0),
                        ]);
                    } else {
                        // Notify user that this container should not be there.
                    }
                } else {
                    $application = resolveContainerOwner($this->applications, $flatLabels, 'application');
                    $applicationId = $application?->id;
                    if ($application) {
                        $foundApplications[] = $application->id;
                        if ($application->container_present !== true) {
                            $application->update(['container_present' => true]);
                        }
                        // Store container status for aggregation
                        if (! isset($this->applicationContainerStatuses)) {
                            $this->applicationContainerStatuses = collect();
                        }
                        if (! $this->applicationContainerStatuses->has($applicationId)) {
                            $this->applicationContainerStatuses->put($applicationId, collect());
                        }
                        $containerName = data_get($labels, 'com.docker.compose.service');
                        // Fallback for Docker Swarm which uses different labels
                        if (! $containerName && $this->server->isSwarm()) {
                            $containerName = data_get($labels, 'coolify.serviceName')
                                ?? data_get($labels, 'coolify.name')
                                ?? data_get($labels, 'com.docker.stack.namespace');
                        }
                        if ($containerName) {
                            $this->applicationContainerStatuses->get($applicationId)->put($containerName, $containerStatus);
                        }

                        // Track restart counts for applications
                        $restartCount = data_get($container, 'RestartCount', 0);
                        if (! isset($this->applicationContainerRestartCounts)) {
                            $this->applicationContainerRestartCounts = collect();
                        }
                        if (! $this->applicationContainerRestartCounts->has($applicationId)) {
                            $this->applicationContainerRestartCounts->put($applicationId, collect());
                        }
                        if ($containerName) {
                            $this->applicationContainerRestartCounts->get($applicationId)->put($containerName, $restartCount);
                        }
                    } else {
                        // Notify user that this container should not be there.
                    }
                }
            } else {
                $uuid = data_get($labels, 'com.docker.compose.service');
                $type = data_get($labels, 'coolify.type');

                if ($uuid) {
                    if ($type === 'service') {
                        if ($serviceSubResource instanceof ServiceDatabase) {
                            $service_db = $serviceSubResource;
                            if ($service_db) {
                                $proxyUuid = $service_db->uuid;
                                $isPublic = data_get($service_db, 'is_public');
                                if ($isPublic) {
                                    $foundTcpProxy = $this->containers->filter(function ($value, $key) use ($proxyUuid) {
                                        if ($this->server->isSwarm()) {
                                            return data_get($value, 'Spec.Name') === "coolify-proxy_$proxyUuid";
                                        } else {
                                            return data_get($value, 'Name') === "/$proxyUuid-proxy";
                                        }
                                    })->first();
                                    if (! $foundTcpProxy) {
                                        StartDatabaseProxy::run($service_db);
                                    } elseif (! $this->server->isSwarm() && StartDatabaseProxy::claimOutdatedRefresh($proxyUuid, data_get($foundTcpProxy, 'Config.Labels'))) {
                                        StartDatabaseProxy::run($service_db);
                                    }
                                } else {
                                    // Clean up orphaned proxy when is_public=false
                                    $orphanedProxy = $this->containers->filter(function ($value, $key) use ($proxyUuid) {
                                        if ($this->server->isSwarm()) {
                                            return data_get($value, 'Spec.Name') === "coolify-proxy_$proxyUuid";
                                        } else {
                                            return data_get($value, 'Name') === "/$proxyUuid-proxy";
                                        }
                                    })->first();
                                    if ($orphanedProxy) {
                                        StopDatabaseProxy::run($service_db);
                                    }
                                }
                            }
                        }
                    } else {
                        $database = $databases->where('uuid', $uuid)->first();
                        if ($database) {
                            $isPublic = data_get($database, 'is_public');
                            $foundDatabases[] = $database->id;
                            $statusFromDb = $database->status;

                            // Track restart count for databases (single-container)
                            $restartCount = data_get($container, 'RestartCount', 0);
                            if ($statusFromDb !== $containerStatus) {
                                $updateData = ['status' => $containerStatus];
                            } else {
                                $updateData = ['last_online_at' => now()];
                            }

                            $database->update($updateData);

                            if ($restartCount > ($database->restart_count ?? 0)) {
                                $database->update([
                                    'restart_count' => (int) $restartCount,
                                    'last_restart_at' => now(),
                                    'last_restart_type' => 'crash',
                                ]);
                            }

                            if ($isPublic) {
                                $foundTcpProxy = $this->containers->filter(function ($value, $key) use ($uuid) {
                                    if ($this->server->isSwarm()) {
                                        return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid";
                                    } else {
                                        return data_get($value, 'Name') === "/$uuid-proxy";
                                    }
                                })->first();
                                if (! $foundTcpProxy) {
                                    StartDatabaseProxy::run($database);
                                } elseif (! $this->server->isSwarm() && StartDatabaseProxy::claimOutdatedRefresh($uuid, data_get($foundTcpProxy, 'Config.Labels'))) {
                                    StartDatabaseProxy::run($database);
                                }
                            } else {
                                // Clean up orphaned proxy when is_public=false
                                $orphanedProxy = $this->containers->filter(function ($value, $key) use ($uuid) {
                                    if ($this->server->isSwarm()) {
                                        return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid";
                                    } else {
                                        return data_get($value, 'Name') === "/$uuid-proxy";
                                    }
                                })->first();
                                if ($orphanedProxy) {
                                    StopDatabaseProxy::run($database);
                                }
                            }
                        } else {
                            // Notify user that this container should not be there.
                        }
                    }
                }
                if (data_get($container, 'Name') === '/coolify-db') {
                    $foundDatabases[] = 0;
                }
            }
            if ($isServiceContainer) {
                if (! $parentService) {
                    continue;
                }
                $serviceLabelId = $parentService->id;
                $subType = $serviceSubType;
                $subId = $serviceSubResource?->id;

                // Store container status for aggregation
                if (! isset($this->serviceContainerStatuses)) {
                    $this->serviceContainerStatuses = collect();
                }

                $key = $serviceLabelId.':'.$subType.':'.$subId;
                if (! $this->serviceContainerStatuses->has($key)) {
                    $this->serviceContainerStatuses->put($key, collect());
                }

                $containerName = data_get($labels, 'com.docker.compose.service');
                if ($containerName) {
                    $this->serviceContainerStatuses->get($key)->put($containerName, $containerStatus);
                    $this->serviceContainerRestartCounts ??= collect();
                    if (! $this->serviceContainerRestartCounts->has($key)) {
                        $this->serviceContainerRestartCounts->put($key, collect());
                    }
                    $this->serviceContainerRestartCounts->get($key)->put($containerName, (int) data_get($container, 'RestartCount', 0));
                }

                // Mark service as found
                $service = $serviceSubResource;
                if ($service) {
                    $foundServices[] = "$service->id-$service->name";
                }
            }
        }
        $exitedServices = collect([]);
        foreach ($services as $service) {
            $apps = $service->applications()->get();
            $dbs = $service->databases()->get();
            foreach ($apps as $app) {
                if (in_array("$app->id-$app->name", $foundServices)) {
                    continue;
                } else {
                    $exitedServices->push($app);
                }
            }
            foreach ($dbs as $db) {
                if (in_array("$db->id-$db->name", $foundServices)) {
                    continue;
                } else {
                    $exitedServices->push($db);
                }
            }
        }
        $exitedServices = $exitedServices->unique('uuid');
        foreach ($exitedServices as $exitedService) {
            if (str($exitedService->status)->startsWith('exited')) {
                continue;
            }

            // Only protection: If no containers at all, Docker query might have failed
            if ($this->containers->isEmpty()) {
                continue;
            }

            if ($exitedService instanceof ServiceDatabase) {
                $exitedService->update(['status' => 'exited']);
            } elseif (! $exitedService->stoppedAfterRestartLimit()) {
                $exitedService->update([
                    'status' => 'exited',
                    'restart_count' => 0,
                    'restart_limit_reached' => false,
                    'last_restart_at' => null,
                    'last_restart_type' => null,
                ]);
            }
        }

        $notRunningApplications = $this->applications->pluck('id')->diff($foundApplications);
        foreach ($notRunningApplications as $applicationId) {
            $application = $this->applications->where('id', $applicationId)->first();

            // Only protection: If no containers at all, Docker query might have failed
            if ($this->containers->isEmpty()) {
                continue;
            }

            if (str($application->status)->startsWith('exited')) {
                $application->update([
                    'container_present' => false,
                    'restart_limit_reached' => false,
                ]);

                continue;
            }

            // If container was recently restarting (crash loop), keep it as degraded for a grace period
            // This prevents false "exited" status during the brief moment between container removal and recreation
            $recentlyRestarted = $application->restart_count > 0 &&
                                 $application->last_restart_at &&
                                 $application->last_restart_at->greaterThan(now()->subSeconds(30));

            if ($recentlyRestarted) {
                // Keep it as degraded if it was recently in a crash loop
                $application->update(['status' => 'degraded:unhealthy']);
            } else {
                // Reset restart count when application exits completely
                $application->update([
                    'status' => 'exited',
                    'container_present' => false,
                    'restart_count' => 0,
                    'last_restart_at' => null,
                    'last_restart_type' => null,
                    'restart_limit_reached' => false,
                ]);
            }
        }
        $notRunningApplicationPreviews = $previews->pluck('id')->diff($foundApplicationPreviews);
        foreach ($notRunningApplicationPreviews as $previewId) {
            $preview = $previews->where('id', $previewId)->first();
            if (str($preview->status)->startsWith('exited')) {
                continue;
            }

            // Only protection: If no containers at all, Docker query might have failed
            if ($this->containers->isEmpty()) {
                continue;
            }

            $preview->update(['status' => 'exited']);
        }
        $notRunningDatabases = $databases->pluck('id')->diff($foundDatabases);
        foreach ($notRunningDatabases as $database) {
            $database = $databases->where('id', $database)->first();
            if (str($database->status)->startsWith('exited')) {
                continue;
            }

            // Only protection: If no containers at all, Docker query might have failed
            if ($this->containers->isEmpty()) {
                continue;
            }

            // Reset restart tracking when database exits completely
            $database->update([
                'status' => 'exited',
                'restart_count' => 0,
                'last_restart_at' => null,
                'last_restart_type' => null,
            ]);

            // Stop proxy if database was public
            if ($database->is_public) {
                StopDatabaseProxy::run($database);
            }

        }

        $this->trackPreviewRestartCounts($previews);

        // Aggregate multi-container application statuses
        if (isset($this->applicationContainerStatuses) && $this->applicationContainerStatuses->isNotEmpty()) {
            foreach ($this->applicationContainerStatuses as $applicationId => $containerStatuses) {
                $application = $this->applications->where('id', $applicationId)->first();
                if (! $application) {
                    continue;
                }

                // Track restart counts first
                $maxRestartCount = 0;
                if (isset($this->applicationContainerRestartCounts) && $this->applicationContainerRestartCounts->has($applicationId)) {
                    $containerRestartCounts = $this->applicationContainerRestartCounts->get($applicationId);
                    $maxRestartCount = $containerRestartCounts->max() ?? 0;
                }

                // Wrap all database updates in a transaction to ensure consistency
                $restartLimitReached = false;

                DB::transaction(function () use ($application, $maxRestartCount, $containerStatuses, &$restartLimitReached) {
                    $previousRestartCount = $application->restart_count ?? 0;
                    $restartState = (new RestartCountTracker)->evaluate(
                        previousRestartCount: $previousRestartCount,
                        observedRestartCount: $maxRestartCount,
                        maxRestartCount: $application->max_restart_count ?? 0,
                    );

                    if ($restartState['restart_count_changed']) {
                        $hasCrashRestarts = $restartState['restart_count'] > 0;
                        $application->update([
                            'restart_count' => $restartState['restart_count'],
                            'last_restart_at' => $hasCrashRestarts ? now() : null,
                            'last_restart_type' => $hasCrashRestarts ? 'crash' : null,
                        ]);
                    }
                    $restartLimitReached = $restartState['restart_limit_reached'];

                    // Aggregate status after tracking restart counts
                    $aggregatedStatus = $this->aggregateApplicationStatus($application, $containerStatuses, $maxRestartCount);
                    if ($aggregatedStatus) {
                        $statusFromDb = $application->status;
                        if ($statusFromDb !== $aggregatedStatus) {
                            $application->update(['status' => $aggregatedStatus]);
                        } else {
                            $application->update(['last_online_at' => now()]);
                        }
                    }
                });

                if ($restartLimitReached) {
                    $restartLimitClaimed = Application::query()
                        ->whereKey($application->getKey())
                        ->where('restart_limit_reached', false)
                        ->update(['restart_limit_reached' => true]) === 1;

                    // Stop again while a container still runs: an earlier stop can fail or come too late.
                    $stillRunning = $containerStatuses->contains(fn (string $status): bool => ! str($status)->contains('exited'));
                    if ($restartLimitClaimed || $stillRunning) {
                        $application->refresh();
                        StopApplication::dispatch(
                            application: $application,
                            previewDeployments: false,
                            dockerCleanup: false,
                            resetRestartCount: false,
                            removeContainers: false,
                        );
                    }
                    if ($restartLimitClaimed) {
                        $application->environment->project->team?->notify(new ApplicationRestartLimitReached($application));
                    }
                }
            }
        }

        // Aggregate multi-container service statuses
        $this->aggregateServiceContainerStatuses($services);

        NotifyResourceStoppedJob::dispatchForStopped($upResourceIds);

        ServiceChecked::dispatch($this->server->team->id);
    }

    private function aggregateApplicationStatus($application, Collection $containerStatuses, int $maxRestartCount = 0): ?string
    {
        return (new ContainerStatusAggregator)->aggregateForCompose($containerStatuses, data_get($application, 'docker_compose_raw'), $maxRestartCount);
    }

    private function aggregateServiceContainerStatuses($services)
    {
        if (! isset($this->serviceContainerStatuses) || $this->serviceContainerStatuses->isEmpty()) {
            return;
        }

        foreach ($this->serviceContainerStatuses as $key => $containerStatuses) {
            // Parse key: serviceId:subType:subId
            [$serviceId, $subType, $subId] = explode(':', $key);

            $service = $services->where('id', $serviceId)->first();
            if (! $service) {
                continue;
            }

            // Get the service sub-resource (ServiceApplication or ServiceDatabase)
            $subResource = null;
            if ($subType === 'application') {
                $subResource = $service->applications()->where('id', $subId)->first();
            } elseif ($subType === 'database') {
                $subResource = $service->databases()->where('id', $subId)->first();
            }

            if (! $subResource) {
                continue;
            }

            $restartCount = isset($this->serviceContainerRestartCounts)
                ? ($this->serviceContainerRestartCounts->get($key)?->max() ?? 0)
                : 0;
            if (! $subResource instanceof ServiceDatabase && $subResource->trackRestartCount($restartCount)) {
                StopServiceApplication::dispatch($subResource, false, false);
                $subResource->team()?->notify(new ApplicationRestartLimitReached($subResource));

                continue;
            }

            $aggregatedStatus = (new ContainerStatusAggregator)->aggregateForCompose($containerStatuses, data_get($service, 'docker_compose_raw'));

            // Update service sub-resource status with aggregated result
            if ($aggregatedStatus) {
                $statusFromDb = $subResource->status;
                if ($statusFromDb !== $aggregatedStatus) {
                    $subResource->update(['status' => $aggregatedStatus]);
                } else {
                    $subResource->update(['last_online_at' => now()]);
                }
            }
        }
    }

    private function trackPreviewRestartCounts(Collection $previews): void
    {
        if (! isset($this->previewContainerRestartCounts)) {
            return;
        }

        $this->previewContainerRestartCounts
            ->groupBy('key')
            ->each(function (Collection $counts, string $key) use ($previews): void {
                [$applicationId, $pullRequestId] = explode(':', $key);
                $preview = $previews->first(fn (ApplicationPreview $preview): bool => (string) $preview->application_id === $applicationId
                    && (string) $preview->pull_request_id === $pullRequestId
                );
                if ($preview?->trackRestartCount((int) $counts->max('count'))) {
                    StopApplicationPreview::dispatch($preview, false, false);
                    $preview->application->environment->project->team?->notify(new ApplicationRestartLimitReached($preview));
                }
            });
    }
}