Files
coolify/app/Actions/Docker/GetContainersStatus.php
T
Andras Bacsai 5a2006e99c perf(docker): look up foreign preview owners once per container status check
GetContainersStatus now resolves all unknown preview owners with one query,
like the Sentinel push. The lookup helper also no longer fails on an
application without a uuid.
2026-09-30 19:23:35 +02:00

660 lines
31 KiB
PHP

<?php
namespace App\Actions\Docker;
use App\Actions\Application\StopApplication;
use App\Actions\Application\StopApplicationPreview;
use App\Actions\Database\StartDatabaseProxy;
use App\Actions\Database\StopDatabaseProxy;
use App\Actions\Service\StopServiceApplication;
use App\Actions\Shared\ComplexStatusCheck;
use App\Events\ServiceChecked;
use App\Models\Application;
use App\Models\ApplicationPreview;
use App\Models\Server;
use App\Models\ServiceDatabase;
use App\Notifications\Application\RestartLimitReached as ApplicationRestartLimitReached;
use App\Services\ContainerStatusAggregator;
use App\Services\RestartCountTracker;
use App\Traits\CalculatesExcludedStatus;
use Illuminate\Support\Arr;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\DB;
use Lorisleiva\Actions\Concerns\AsAction;
class GetContainersStatus
{
use AsAction;
use CalculatesExcludedStatus;
public string $jobQueue = 'high';
public $applications;
public ?Collection $containers;
public ?Collection $containerReplicates;
public $server;
protected ?Collection $applicationContainerStatuses;
protected ?Collection $applicationContainerRestartCounts;
protected ?Collection $previewContainerRestartCounts;
protected ?Collection $serviceContainerStatuses;
protected ?Collection $serviceContainerRestartCounts;
public function handle(Server $server, ?Collection $containers = null, ?Collection $containerReplicates = null)
{
$this->containers = $containers;
$this->containerReplicates = $containerReplicates;
$this->server = $server;
if (! $this->server->isFunctional()) {
return 'Server is not functional.';
}
$this->applications = $this->server->applications();
$skip_these_applications = collect([]);
foreach ($this->applications as $application) {
if ($application->additional_servers->count() > 0) {
$skip_these_applications->push($application);
ComplexStatusCheck::run($application);
$this->applications = $this->applications->filter(function ($value, $key) use ($application) {
return $value->id !== $application->id;
});
}
}
$this->applications = $this->applications->filter(function ($value, $key) use ($skip_these_applications) {
return ! $skip_these_applications->pluck('id')->contains($value->id);
});
if ($this->containers === null) {
['containers' => $this->containers, 'containerReplicates' => $this->containerReplicates] = $this->server->getContainers();
}
if (is_null($this->containers)) {
return;
}
if ($this->containerReplicates) {
foreach ($this->containerReplicates as $containerReplica) {
$name = data_get($containerReplica, 'Name');
$this->containers = $this->containers->map(function ($container) use ($name, $containerReplica) {
if (data_get($container, 'Spec.Name') === $name) {
$replicas = data_get($containerReplica, 'Replicas');
$running = str($replicas)->explode('/')[0];
$total = str($replicas)->explode('/')[1];
if ($running === $total) {
data_set($container, 'State.Status', 'running');
data_set($container, 'State.Health.Status', 'healthy');
} else {
data_set($container, 'State.Status', 'starting');
data_set($container, 'State.Health.Status', 'unhealthy');
}
}
return $container;
});
}
}
$databases = $this->server->databases();
$services = $this->server->services()->get();
$previews = $this->server->previews();
$foundApplications = [];
$foundApplicationPreviews = [];
$foundDatabases = [];
$foundServices = [];
// Owners of preview containers outside this server's applications, in one query.
$foreignApplicationIdsByUuid = containerApplicationIdsByUuid(
$this->applications,
$this->containers
->map(fn ($container) => format_docker_labels_to_json(data_get($container, $this->server->isSwarm() ? 'Spec.Labels' : 'Config.Labels') ?? []))
->filter(fn (Collection $labels) => (bool) $labels->get('coolify.pullRequestId') && isContainerOfType($labels, 'application'))
);
foreach ($this->containers as $container) {
if ($this->server->isSwarm()) {
$labels = data_get($container, 'Spec.Labels');
$uuid = data_get($labels, 'coolify.name');
} else {
$labels = data_get($container, 'Config.Labels');
}
$containerStatus = data_get($container, 'State.Status');
$containerHealth = data_get($container, 'State.Health.Status');
if ($containerStatus === 'restarting') {
$healthSuffix = $containerHealth ?? 'unknown';
$containerStatus = "restarting:$healthSuffix";
} elseif ($containerStatus === 'exited') {
// Keep as-is, no health suffix for exited containers
} else {
$healthSuffix = $containerHealth ?? 'unknown';
$containerStatus = "$containerStatus:$healthSuffix";
}
$flatLabels = format_docker_labels_to_json($labels);
$labels = Arr::undot($flatLabels);
if (filter_var(data_get($labels, 'com.docker.compose.oneoff'), FILTER_VALIDATE_BOOLEAN)) {
continue;
}
// Containers are matched by owner UUID; numeric ids differ between instances.
$isApplicationContainer = isContainerOfType($flatLabels, 'application');
$isServiceContainer = isContainerOfType($flatLabels, 'service');
[$parentService, $serviceSubType, $serviceSubResource] = resolveServiceContainerOwner($services, $flatLabels);
if ($isApplicationContainer) {
$pullRequestId = data_get($labels, 'coolify.pullRequestId');
if ($pullRequestId) {
$applicationId = resolveContainerApplicationId($this->applications, $flatLabels, $foreignApplicationIdsByUuid);
$preview = $applicationId
? ApplicationPreview::where('application_id', $applicationId)->where('pull_request_id', $pullRequestId)->first()
: null;
if ($preview) {
$foundApplicationPreviews[] = $preview->id;
$statusFromDb = $preview->status;
if ($statusFromDb !== $containerStatus) {
$preview->update(['status' => $containerStatus]);
} else {
$preview->update(['last_online_at' => now()]);
}
$key = $applicationId.':'.$pullRequestId;
$this->previewContainerRestartCounts ??= collect();
$this->previewContainerRestartCounts->push([
'key' => $key,
'count' => (int) data_get($container, 'RestartCount', 0),
]);
} else {
// Notify user that this container should not be there.
}
} else {
$application = resolveContainerOwner($this->applications, $flatLabels, 'application');
$applicationId = $application?->id;
if ($application) {
$foundApplications[] = $application->id;
if ($application->container_present !== true) {
$application->update(['container_present' => true]);
}
// Store container status for aggregation
if (! isset($this->applicationContainerStatuses)) {
$this->applicationContainerStatuses = collect();
}
if (! $this->applicationContainerStatuses->has($applicationId)) {
$this->applicationContainerStatuses->put($applicationId, collect());
}
$containerName = data_get($labels, 'com.docker.compose.service');
// Fallback for Docker Swarm which uses different labels
if (! $containerName && $this->server->isSwarm()) {
$containerName = data_get($labels, 'coolify.serviceName')
?? data_get($labels, 'coolify.name')
?? data_get($labels, 'com.docker.stack.namespace');
}
if ($containerName) {
$this->applicationContainerStatuses->get($applicationId)->put($containerName, $containerStatus);
}
// Track restart counts for applications
$restartCount = data_get($container, 'RestartCount', 0);
if (! isset($this->applicationContainerRestartCounts)) {
$this->applicationContainerRestartCounts = collect();
}
if (! $this->applicationContainerRestartCounts->has($applicationId)) {
$this->applicationContainerRestartCounts->put($applicationId, collect());
}
if ($containerName) {
$this->applicationContainerRestartCounts->get($applicationId)->put($containerName, $restartCount);
}
} else {
// Notify user that this container should not be there.
}
}
} else {
$uuid = data_get($labels, 'com.docker.compose.service');
$type = data_get($labels, 'coolify.type');
if ($uuid) {
if ($type === 'service') {
if ($serviceSubResource instanceof ServiceDatabase) {
$service_db = $serviceSubResource;
if ($service_db) {
$proxyUuid = $service_db->uuid;
$isPublic = data_get($service_db, 'is_public');
if ($isPublic) {
$foundTcpProxy = $this->containers->filter(function ($value, $key) use ($proxyUuid) {
if ($this->server->isSwarm()) {
return data_get($value, 'Spec.Name') === "coolify-proxy_$proxyUuid";
} else {
return data_get($value, 'Name') === "/$proxyUuid-proxy";
}
})->first();
if (! $foundTcpProxy) {
StartDatabaseProxy::run($service_db);
}
} else {
// Clean up orphaned proxy when is_public=false
$orphanedProxy = $this->containers->filter(function ($value, $key) use ($proxyUuid) {
if ($this->server->isSwarm()) {
return data_get($value, 'Spec.Name') === "coolify-proxy_$proxyUuid";
} else {
return data_get($value, 'Name') === "/$proxyUuid-proxy";
}
})->first();
if ($orphanedProxy) {
StopDatabaseProxy::run($service_db);
}
}
}
}
} else {
$database = $databases->where('uuid', $uuid)->first();
if ($database) {
$isPublic = data_get($database, 'is_public');
$foundDatabases[] = $database->id;
$statusFromDb = $database->status;
// Track restart count for databases (single-container)
$restartCount = data_get($container, 'RestartCount', 0);
if ($statusFromDb !== $containerStatus) {
$updateData = ['status' => $containerStatus];
} else {
$updateData = ['last_online_at' => now()];
}
$database->update($updateData);
if ($restartCount > ($database->restart_count ?? 0)) {
$database->update([
'restart_count' => (int) $restartCount,
'last_restart_at' => now(),
'last_restart_type' => 'crash',
]);
}
if ($isPublic) {
$foundTcpProxy = $this->containers->filter(function ($value, $key) use ($uuid) {
if ($this->server->isSwarm()) {
return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid";
} else {
return data_get($value, 'Name') === "/$uuid-proxy";
}
})->first();
if (! $foundTcpProxy) {
StartDatabaseProxy::run($database);
}
} else {
// Clean up orphaned proxy when is_public=false
$orphanedProxy = $this->containers->filter(function ($value, $key) use ($uuid) {
if ($this->server->isSwarm()) {
return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid";
} else {
return data_get($value, 'Name') === "/$uuid-proxy";
}
})->first();
if ($orphanedProxy) {
StopDatabaseProxy::run($database);
}
}
} else {
// Notify user that this container should not be there.
}
}
}
if (data_get($container, 'Name') === '/coolify-db') {
$foundDatabases[] = 0;
}
}
if ($isServiceContainer) {
if (! $parentService) {
continue;
}
$serviceLabelId = $parentService->id;
$subType = $serviceSubType;
$subId = $serviceSubResource?->id;
// Store container status for aggregation
if (! isset($this->serviceContainerStatuses)) {
$this->serviceContainerStatuses = collect();
}
$key = $serviceLabelId.':'.$subType.':'.$subId;
if (! $this->serviceContainerStatuses->has($key)) {
$this->serviceContainerStatuses->put($key, collect());
}
$containerName = data_get($labels, 'com.docker.compose.service');
if ($containerName) {
$this->serviceContainerStatuses->get($key)->put($containerName, $containerStatus);
$this->serviceContainerRestartCounts ??= collect();
if (! $this->serviceContainerRestartCounts->has($key)) {
$this->serviceContainerRestartCounts->put($key, collect());
}
$this->serviceContainerRestartCounts->get($key)->put($containerName, (int) data_get($container, 'RestartCount', 0));
}
// Mark service as found
$service = $serviceSubResource;
if ($service) {
$foundServices[] = "$service->id-$service->name";
}
}
}
$exitedServices = collect([]);
foreach ($services as $service) {
$apps = $service->applications()->get();
$dbs = $service->databases()->get();
foreach ($apps as $app) {
if (in_array("$app->id-$app->name", $foundServices)) {
continue;
} else {
$exitedServices->push($app);
}
}
foreach ($dbs as $db) {
if (in_array("$db->id-$db->name", $foundServices)) {
continue;
} else {
$exitedServices->push($db);
}
}
}
$exitedServices = $exitedServices->unique('uuid');
foreach ($exitedServices as $exitedService) {
if (str($exitedService->status)->startsWith('exited')) {
continue;
}
// Only protection: If no containers at all, Docker query might have failed
if ($this->containers->isEmpty()) {
continue;
}
if ($exitedService instanceof ServiceDatabase) {
$exitedService->update(['status' => 'exited']);
} elseif (! $exitedService->stoppedAfterRestartLimit()) {
$exitedService->update([
'status' => 'exited',
'restart_count' => 0,
'restart_limit_reached' => false,
'last_restart_at' => null,
'last_restart_type' => null,
]);
}
}
$notRunningApplications = $this->applications->pluck('id')->diff($foundApplications);
foreach ($notRunningApplications as $applicationId) {
$application = $this->applications->where('id', $applicationId)->first();
// Only protection: If no containers at all, Docker query might have failed
if ($this->containers->isEmpty()) {
continue;
}
if (str($application->status)->startsWith('exited')) {
$application->update([
'container_present' => false,
'restart_limit_reached' => false,
]);
continue;
}
// If container was recently restarting (crash loop), keep it as degraded for a grace period
// This prevents false "exited" status during the brief moment between container removal and recreation
$recentlyRestarted = $application->restart_count > 0 &&
$application->last_restart_at &&
$application->last_restart_at->greaterThan(now()->subSeconds(30));
if ($recentlyRestarted) {
// Keep it as degraded if it was recently in a crash loop
$application->update(['status' => 'degraded:unhealthy']);
} else {
// Reset restart count when application exits completely
$application->update([
'status' => 'exited',
'container_present' => false,
'restart_count' => 0,
'last_restart_at' => null,
'last_restart_type' => null,
'restart_limit_reached' => false,
]);
}
}
$notRunningApplicationPreviews = $previews->pluck('id')->diff($foundApplicationPreviews);
foreach ($notRunningApplicationPreviews as $previewId) {
$preview = $previews->where('id', $previewId)->first();
if (str($preview->status)->startsWith('exited')) {
continue;
}
// Only protection: If no containers at all, Docker query might have failed
if ($this->containers->isEmpty()) {
continue;
}
$preview->update(['status' => 'exited']);
}
$notRunningDatabases = $databases->pluck('id')->diff($foundDatabases);
foreach ($notRunningDatabases as $database) {
$database = $databases->where('id', $database)->first();
if (str($database->status)->startsWith('exited')) {
continue;
}
// Only protection: If no containers at all, Docker query might have failed
if ($this->containers->isEmpty()) {
continue;
}
// Reset restart tracking when database exits completely
$database->update([
'status' => 'exited',
'restart_count' => 0,
'last_restart_at' => null,
'last_restart_type' => null,
]);
// Stop proxy if database was public
if ($database->is_public) {
StopDatabaseProxy::run($database);
}
}
$this->trackPreviewRestartCounts($previews);
// Aggregate multi-container application statuses
if (isset($this->applicationContainerStatuses) && $this->applicationContainerStatuses->isNotEmpty()) {
foreach ($this->applicationContainerStatuses as $applicationId => $containerStatuses) {
$application = $this->applications->where('id', $applicationId)->first();
if (! $application) {
continue;
}
// Track restart counts first
$maxRestartCount = 0;
if (isset($this->applicationContainerRestartCounts) && $this->applicationContainerRestartCounts->has($applicationId)) {
$containerRestartCounts = $this->applicationContainerRestartCounts->get($applicationId);
$maxRestartCount = $containerRestartCounts->max() ?? 0;
}
// Wrap all database updates in a transaction to ensure consistency
$restartLimitReached = false;
DB::transaction(function () use ($application, $maxRestartCount, $containerStatuses, &$restartLimitReached) {
$previousRestartCount = $application->restart_count ?? 0;
$restartState = (new RestartCountTracker)->evaluate(
previousRestartCount: $previousRestartCount,
observedRestartCount: $maxRestartCount,
maxRestartCount: $application->max_restart_count ?? 0,
);
if ($restartState['restart_count_changed']) {
$hasCrashRestarts = $restartState['restart_count'] > 0;
$application->update([
'restart_count' => $restartState['restart_count'],
'last_restart_at' => $hasCrashRestarts ? now() : null,
'last_restart_type' => $hasCrashRestarts ? 'crash' : null,
]);
}
$restartLimitReached = $restartState['restart_limit_reached'];
// Aggregate status after tracking restart counts
$aggregatedStatus = $this->aggregateApplicationStatus($application, $containerStatuses, $maxRestartCount);
if ($aggregatedStatus) {
$statusFromDb = $application->status;
if ($statusFromDb !== $aggregatedStatus) {
$application->update(['status' => $aggregatedStatus]);
} else {
$application->update(['last_online_at' => now()]);
}
}
});
if ($restartLimitReached) {
$restartLimitClaimed = Application::query()
->whereKey($application->getKey())
->where('restart_limit_reached', false)
->update(['restart_limit_reached' => true]) === 1;
if ($restartLimitClaimed) {
$application->refresh();
StopApplication::dispatch(
application: $application,
previewDeployments: false,
dockerCleanup: false,
resetRestartCount: false,
removeContainers: false,
);
$application->environment->project->team?->notify(new ApplicationRestartLimitReached($application));
}
}
}
}
// Aggregate multi-container service statuses
$this->aggregateServiceContainerStatuses($services);
ServiceChecked::dispatch($this->server->team->id);
}
private function aggregateApplicationStatus($application, Collection $containerStatuses, int $maxRestartCount = 0): ?string
{
// Parse docker compose to check for excluded containers
$dockerComposeRaw = data_get($application, 'docker_compose_raw');
$excludedContainers = $this->getExcludedContainersFromDockerCompose($dockerComposeRaw);
// Filter out excluded containers
$relevantStatuses = $containerStatuses->filter(function ($status, $containerName) use ($excludedContainers) {
return ! $excludedContainers->contains($containerName);
});
// If all containers are excluded, calculate status from excluded containers
if ($relevantStatuses->isEmpty()) {
return $this->calculateExcludedStatusFromStrings($containerStatuses);
}
// Use ContainerStatusAggregator service for state machine logic
// Use preserveRestarting: true so applications show "Restarting" instead of "Degraded"
$aggregator = new ContainerStatusAggregator;
return $aggregator->aggregateFromStrings($relevantStatuses, $maxRestartCount, preserveRestarting: true);
}
private function aggregateServiceContainerStatuses($services)
{
if (! isset($this->serviceContainerStatuses) || $this->serviceContainerStatuses->isEmpty()) {
return;
}
foreach ($this->serviceContainerStatuses as $key => $containerStatuses) {
// Parse key: serviceId:subType:subId
[$serviceId, $subType, $subId] = explode(':', $key);
$service = $services->where('id', $serviceId)->first();
if (! $service) {
continue;
}
// Get the service sub-resource (ServiceApplication or ServiceDatabase)
$subResource = null;
if ($subType === 'application') {
$subResource = $service->applications()->where('id', $subId)->first();
} elseif ($subType === 'database') {
$subResource = $service->databases()->where('id', $subId)->first();
}
if (! $subResource) {
continue;
}
$restartCount = isset($this->serviceContainerRestartCounts)
? ($this->serviceContainerRestartCounts->get($key)?->max() ?? 0)
: 0;
if (! $subResource instanceof ServiceDatabase && $subResource->trackRestartCount($restartCount)) {
StopServiceApplication::dispatch($subResource, false, false);
$subResource->team()?->notify(new ApplicationRestartLimitReached($subResource));
continue;
}
// Parse docker compose from service to check for excluded containers
$dockerComposeRaw = data_get($service, 'docker_compose_raw');
$excludedContainers = $this->getExcludedContainersFromDockerCompose($dockerComposeRaw);
// Filter out excluded containers
$relevantStatuses = $containerStatuses->filter(function ($status, $containerName) use ($excludedContainers) {
return ! $excludedContainers->contains($containerName);
});
// If all containers are excluded, calculate status from excluded containers
if ($relevantStatuses->isEmpty()) {
$aggregatedStatus = $this->calculateExcludedStatusFromStrings($containerStatuses);
if ($aggregatedStatus) {
$statusFromDb = $subResource->status;
if ($statusFromDb !== $aggregatedStatus) {
$subResource->update(['status' => $aggregatedStatus]);
} else {
$subResource->update(['last_online_at' => now()]);
}
}
continue;
}
// Use ContainerStatusAggregator service for state machine logic
// Use preserveRestarting: true so individual sub-resources show "Restarting" instead of "Degraded"
$aggregator = new ContainerStatusAggregator;
$aggregatedStatus = $aggregator->aggregateFromStrings($relevantStatuses, preserveRestarting: true);
// Update service sub-resource status with aggregated result
if ($aggregatedStatus) {
$statusFromDb = $subResource->status;
if ($statusFromDb !== $aggregatedStatus) {
$subResource->update(['status' => $aggregatedStatus]);
} else {
$subResource->update(['last_online_at' => now()]);
}
}
}
}
private function trackPreviewRestartCounts(Collection $previews): void
{
if (! isset($this->previewContainerRestartCounts)) {
return;
}
$this->previewContainerRestartCounts
->groupBy('key')
->each(function (Collection $counts, string $key) use ($previews): void {
[$applicationId, $pullRequestId] = explode(':', $key);
$preview = $previews->first(fn (ApplicationPreview $preview): bool => (string) $preview->application_id === $applicationId
&& (string) $preview->pull_request_id === $pullRequestId
);
if ($preview?->trackRestartCount((int) $counts->max('count'))) {
StopApplicationPreview::dispatch($preview, false, false);
$preview->application->environment->project->team?->notify(new ApplicationRestartLimitReached($preview));
}
});
}
}