starred/coolify
1
0
Fork 0
mirror of https://github.com/coollabsio/coolify.git synced 2026-09-27 17:55:59 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'next' into env-var-autocomplete

Browse Source
This commit is contained in:
Andras Bacsai
2025-11-25 09:35:20 +01:00
committed by GitHub
parent c79b5f1e5c 99e8aeb6f8
commit 01d3f07934
105 changed files with 7956 additions and 2732 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Actions/Database/RestartDatabase.php
+1 -1
View File
@@ -22,7 +22,7 @@ class RestartDatabase
if (! $server->isFunctional()) {
return 'Server is not functional';
}
StopDatabase::run($database);
StopDatabase::run($database, dockerCleanup: false);
return StartDatabase::run($database);
}
app/Actions/Docker/GetContainersStatus.php
+107 -70
View File
@@ -8,6 +8,8 @@ use App\Events\ServiceChecked;
use App\Models\ApplicationPreview;
use App\Models\Server;
use App\Models\ServiceDatabase;
use App\Services\ContainerStatusAggregator;
use App\Traits\CalculatesExcludedStatus;
use Illuminate\Support\Arr;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\DB;
@@ -16,6 +18,7 @@ use Lorisleiva\Actions\Concerns\AsAction;
class GetContainersStatus
{
use AsAction;
use CalculatesExcludedStatus;
public string $jobQueue = 'high';
@@ -31,6 +34,8 @@ class GetContainersStatus
protected ?Collection $applicationContainerRestartCounts;
protected ?Collection $serviceContainerStatuses;
public function handle(Server $server, ?Collection $containers = null, ?Collection $containerReplicates = null)
{
$this->containers = $containers;
@@ -98,11 +103,15 @@ class GetContainersStatus
$labels = data_get($container, 'Config.Labels');
}
$containerStatus = data_get($container, 'State.Status');
$containerHealth = data_get($container, 'State.Health.Status', 'unhealthy');
$containerHealth = data_get($container, 'State.Health.Status');
if ($containerStatus === 'restarting') {
$containerStatus = "restarting ($containerHealth)";
$healthSuffix = $containerHealth ?? 'unknown';
$containerStatus = "restarting:$healthSuffix";
} elseif ($containerStatus === 'exited') {
// Keep as-is, no health suffix for exited containers
} else {
$containerStatus = "$containerStatus ($containerHealth)";
$healthSuffix = $containerHealth ?? 'unknown';
$containerStatus = "$containerStatus:$healthSuffix";
}
$labels = Arr::undot(format_docker_labels_to_json($labels));
$applicationId = data_get($labels, 'coolify.applicationId');
@@ -222,23 +231,34 @@ class GetContainersStatus
if ($serviceLabelId) {
$subType = data_get($labels, 'coolify.service.subType');
$subId = data_get($labels, 'coolify.service.subId');
$service = $services->where('id', $serviceLabelId)->first();
if (! $service) {
$parentService = $services->where('id', $serviceLabelId)->first();
if (! $parentService) {
continue;
}
// Store container status for aggregation
if (! isset($this->serviceContainerStatuses)) {
$this->serviceContainerStatuses = collect();
}
$key = $serviceLabelId.':'.$subType.':'.$subId;
if (! $this->serviceContainerStatuses->has($key)) {
$this->serviceContainerStatuses->put($key, collect());
}
$containerName = data_get($labels, 'com.docker.compose.service');
if ($containerName) {
$this->serviceContainerStatuses->get($key)->put($containerName, $containerStatus);
}
// Mark service as found
if ($subType === 'application') {
$service = $service->applications()->where('id', $subId)->first();
$service = $parentService->applications()->where('id', $subId)->first();
} else {
$service = $service->databases()->where('id', $subId)->first();
$service = $parentService->databases()->where('id', $subId)->first();
}
if ($service) {
$foundServices[] = "$service->id-$service->name";
$statusFromDb = $service->status;
if ($statusFromDb !== $containerStatus) {
$service->update(['status' => $containerStatus]);
} else {
$service->update(['last_online_at' => now()]);
}
}
}
}
@@ -314,7 +334,7 @@ class GetContainersStatus
if ($recentlyRestarted) {
// Keep it as degraded if it was recently in a crash loop
$application->update(['status' => 'degraded (unhealthy)']);
$application->update(['status' => 'degraded:unhealthy']);
} else {
// Reset restart count when application exits completely
$application->update([
@@ -418,6 +438,9 @@ class GetContainersStatus
}
}
// Aggregate multi-container service statuses
$this->aggregateServiceContainerStatuses($services);
ServiceChecked::dispatch($this->server->team->id);
}
@@ -425,74 +448,88 @@ class GetContainersStatus
{
// Parse docker compose to check for excluded containers
$dockerComposeRaw = data_get($application, 'docker_compose_raw');
$excludedContainers = collect();
if ($dockerComposeRaw) {
try {
$dockerCompose = \Symfony\Component\Yaml\Yaml::parse($dockerComposeRaw);
$services = data_get($dockerCompose, 'services', []);
foreach ($services as $serviceName => $serviceConfig) {
// Check if container should be excluded
$excludeFromHc = data_get($serviceConfig, 'exclude_from_hc', false);
$restartPolicy = data_get($serviceConfig, 'restart', 'always');
if ($excludeFromHc || $restartPolicy === 'no') {
$excludedContainers->push($serviceName);
}
}
} catch (\Exception $e) {
// If we can't parse, treat all containers as included
}
}
$excludedContainers = $this->getExcludedContainersFromDockerCompose($dockerComposeRaw);
// Filter out excluded containers
$relevantStatuses = $containerStatuses->filter(function ($status, $containerName) use ($excludedContainers) {
return ! $excludedContainers->contains($containerName);
});
// If all containers are excluded, don't update status
// If all containers are excluded, calculate status from excluded containers
if ($relevantStatuses->isEmpty()) {
return null;
return $this->calculateExcludedStatusFromStrings($containerStatuses);
}
$hasRunning = false;
$hasRestarting = false;
$hasUnhealthy = false;
$hasExited = false;
// Use ContainerStatusAggregator service for state machine logic
$aggregator = new ContainerStatusAggregator;
foreach ($relevantStatuses as $status) {
if (str($status)->contains('restarting')) {
$hasRestarting = true;
} elseif (str($status)->contains('running')) {
$hasRunning = true;
if (str($status)->contains('unhealthy')) {
$hasUnhealthy = true;
return $aggregator->aggregateFromStrings($relevantStatuses, $maxRestartCount);
}
private function aggregateServiceContainerStatuses($services)
{
if (! isset($this->serviceContainerStatuses) || $this->serviceContainerStatuses->isEmpty()) {
return;
}
foreach ($this->serviceContainerStatuses as $key => $containerStatuses) {
// Parse key: serviceId:subType:subId
[$serviceId, $subType, $subId] = explode(':', $key);
$service = $services->where('id', $serviceId)->first();
if (! $service) {
continue;
}
// Get the service sub-resource (ServiceApplication or ServiceDatabase)
$subResource = null;
if ($subType === 'application') {
$subResource = $service->applications()->where('id', $subId)->first();
} elseif ($subType === 'database') {
$subResource = $service->databases()->where('id', $subId)->first();
}
if (! $subResource) {
continue;
}
// Parse docker compose from service to check for excluded containers
$dockerComposeRaw = data_get($service, 'docker_compose_raw');
$excludedContainers = $this->getExcludedContainersFromDockerCompose($dockerComposeRaw);
// Filter out excluded containers
$relevantStatuses = $containerStatuses->filter(function ($status, $containerName) use ($excludedContainers) {
return ! $excludedContainers->contains($containerName);
});
// If all containers are excluded, calculate status from excluded containers
if ($relevantStatuses->isEmpty()) {
$aggregatedStatus = $this->calculateExcludedStatusFromStrings($containerStatuses);
if ($aggregatedStatus) {
$statusFromDb = $subResource->status;
if ($statusFromDb !== $aggregatedStatus) {
$subResource->update(['status' => $aggregatedStatus]);
} else {
$subResource->update(['last_online_at' => now()]);
}
}
continue;
}
// Use ContainerStatusAggregator service for state machine logic
$aggregator = new ContainerStatusAggregator;
$aggregatedStatus = $aggregator->aggregateFromStrings($relevantStatuses);
// Update service sub-resource status with aggregated result
if ($aggregatedStatus) {
$statusFromDb = $subResource->status;
if ($statusFromDb !== $aggregatedStatus) {
$subResource->update(['status' => $aggregatedStatus]);
} else {
$subResource->update(['last_online_at' => now()]);
}
} elseif (str($status)->contains('exited')) {
$hasExited = true;
$hasUnhealthy = true;
}
}
if ($hasRestarting) {
return 'degraded (unhealthy)';
}
// If container is exited but has restart count > 0, it's in a crash loop
if ($hasExited && $maxRestartCount > 0) {
return 'degraded (unhealthy)';
}
if ($hasRunning && $hasExited) {
return 'degraded (unhealthy)';
}
if ($hasRunning) {
return $hasUnhealthy ? 'running (unhealthy)' : 'running (healthy)';
}
// All containers are exited with no restart count - truly stopped
return 'exited (unhealthy)';
}
}
app/Actions/Server/InstallDocker.php
-31
View File
@@ -59,8 +59,6 @@ class InstallDocker
$command = collect([]);
if (isDev() && $server->id === 0) {
$command = $command->merge([
"echo 'Installing Prerequisites...'",
'sleep 1',
"echo 'Installing Docker Engine...'",
"echo 'Configuring Docker Engine (merging existing configuration with the required)...'",
'sleep 4',
@@ -70,35 +68,6 @@ class InstallDocker
return remote_process($command, $server);
} else {
if ($supported_os_type->contains('debian')) {
$command = $command->merge([
"echo 'Installing Prerequisites...'",
'apt-get update -y',
'command -v curl >/dev/null || apt install -y curl',
'command -v wget >/dev/null || apt install -y wget',
'command -v git >/dev/null || apt install -y git',
'command -v jq >/dev/null || apt install -y jq',
]);
} elseif ($supported_os_type->contains('rhel')) {
$command = $command->merge([
"echo 'Installing Prerequisites...'",
'command -v curl >/dev/null || dnf install -y curl',
'command -v wget >/dev/null || dnf install -y wget',
'command -v git >/dev/null || dnf install -y git',
'command -v jq >/dev/null || dnf install -y jq',
]);
} elseif ($supported_os_type->contains('sles')) {
$command = $command->merge([
"echo 'Installing Prerequisites...'",
'zypper update -y',
'command -v curl >/dev/null || zypper install -y curl',
'command -v wget >/dev/null || zypper install -y wget',
'command -v git >/dev/null || zypper install -y git',
'command -v jq >/dev/null || zypper install -y jq',
]);
} else {
throw new \Exception('Unsupported OS');
}
$command = $command->merge([
"echo 'Installing Docker Engine...'",
]);
app/Actions/Server/InstallPrerequisites.php
+57
View File
@@ -0,0 +1,57 @@
<?php
namespace App\Actions\Server;
use App\Models\Server;
use Lorisleiva\Actions\Concerns\AsAction;
class InstallPrerequisites
{
use AsAction;
public string $jobQueue = 'high';
public function handle(Server $server)
{
$supported_os_type = $server->validateOS();
if (! $supported_os_type) {
throw new \Exception('Server OS type is not supported for automated installation. Please install prerequisites manually.');
}
$command = collect([]);
if ($supported_os_type->contains('debian')) {
$command = $command->merge([
"echo 'Installing Prerequisites...'",
'apt-get update -y',
'command -v curl >/dev/null || apt install -y curl',
'command -v wget >/dev/null || apt install -y wget',
'command -v git >/dev/null || apt install -y git',
'command -v jq >/dev/null || apt install -y jq',
]);
} elseif ($supported_os_type->contains('rhel')) {
$command = $command->merge([
"echo 'Installing Prerequisites...'",
'command -v curl >/dev/null || dnf install -y curl',
'command -v wget >/dev/null || dnf install -y wget',
'command -v git >/dev/null || dnf install -y git',
'command -v jq >/dev/null || dnf install -y jq',
]);
} elseif ($supported_os_type->contains('sles')) {
$command = $command->merge([
"echo 'Installing Prerequisites...'",
'zypper update -y',
'command -v curl >/dev/null || zypper install -y curl',
'command -v wget >/dev/null || zypper install -y wget',
'command -v git >/dev/null || zypper install -y git',
'command -v jq >/dev/null || zypper install -y jq',
]);
} else {
throw new \Exception('Unsupported OS type for prerequisites installation');
}
$command->push("echo 'Prerequisites installed successfully.'");
return remote_process($command, $server);
}
}
app/Actions/Server/ValidatePrerequisites.php
+40
View File
@@ -0,0 +1,40 @@
<?php
namespace App\Actions\Server;
use App\Models\Server;
use Lorisleiva\Actions\Concerns\AsAction;
class ValidatePrerequisites
{
use AsAction;
public string $jobQueue = 'high';
/**
* Validate that required commands are available on the server.
*
* @return array{success: bool, missing: array<string>, found: array<string>}
*/
public function handle(Server $server): array
{
$requiredCommands = ['git', 'curl', 'jq'];
$missing = [];
$found = [];
foreach ($requiredCommands as $cmd) {
$result = instant_remote_process(["command -v {$cmd}"], $server, false);
if (! $result) {
$missing[] = $cmd;
} else {
$found[] = $cmd;
}
}
return [
'success' => empty($missing),
'missing' => $missing,
'found' => $found,
];
}
}
app/Actions/Server/ValidateServer.php
+10
View File
@@ -45,6 +45,16 @@ class ValidateServer
throw new \Exception($this->error);
}
$validationResult = $server->validatePrerequisites();
if (! $validationResult['success']) {
$missingCommands = implode(', ', $validationResult['missing']);
$this->error = "Prerequisites ({$missingCommands}) are not installed. Please install them before continuing or use the validation with installation endpoint.";
$server->update([
'validation_logs' => $this->error,
]);
throw new \Exception($this->error);
}
$this->docker_installed = $server->validateDockerEngine();
$this->docker_compose_installed = $server->validateDockerCompose();
if (! $this->docker_installed || ! $this->docker_compose_installed) {
app/Actions/Shared/ComplexStatusCheck.php
+19 -65
View File
@@ -3,11 +3,14 @@
namespace App\Actions\Shared;
use App\Models\Application;
use App\Services\ContainerStatusAggregator;
use App\Traits\CalculatesExcludedStatus;
use Lorisleiva\Actions\Concerns\AsAction;
class ComplexStatusCheck
{
use AsAction;
use CalculatesExcludedStatus;
public function handle(Application $application)
{
@@ -17,11 +20,11 @@ class ComplexStatusCheck
$is_main_server = $application->destination->server->id === $server->id;
if (! $server->isFunctional()) {
if ($is_main_server) {
$application->update(['status' => 'exited:unhealthy']);
$application->update(['status' => 'exited']);
continue;
} else {
$application->additional_servers()->updateExistingPivot($server->id, ['status' => 'exited:unhealthy']);
$application->additional_servers()->updateExistingPivot($server->id, ['status' => 'exited']);
continue;
}
@@ -46,11 +49,11 @@ class ComplexStatusCheck
}
} else {
if ($is_main_server) {
$application->update(['status' => 'exited:unhealthy']);
$application->update(['status' => 'exited']);
continue;
} else {
$application->additional_servers()->updateExistingPivot($server->id, ['status' => 'exited:unhealthy']);
$application->additional_servers()->updateExistingPivot($server->id, ['status' => 'exited']);
continue;
}
@@ -61,74 +64,25 @@ class ComplexStatusCheck
private function aggregateContainerStatuses($application, $containers)
{
$dockerComposeRaw = data_get($application, 'docker_compose_raw');
$excludedContainers = collect();
$excludedContainers = $this->getExcludedContainersFromDockerCompose($dockerComposeRaw);
if ($dockerComposeRaw) {
try {
$dockerCompose = \Symfony\Component\Yaml\Yaml::parse($dockerComposeRaw);
$services = data_get($dockerCompose, 'services', []);
foreach ($services as $serviceName => $serviceConfig) {
$excludeFromHc = data_get($serviceConfig, 'exclude_from_hc', false);
$restartPolicy = data_get($serviceConfig, 'restart', 'always');
if ($excludeFromHc || $restartPolicy === 'no') {
$excludedContainers->push($serviceName);
}
}
} catch (\Exception $e) {
// If we can't parse, treat all containers as included
}
}
$hasRunning = false;
$hasRestarting = false;
$hasUnhealthy = false;
$hasExited = false;
$relevantContainerCount = 0;
foreach ($containers as $container) {
// Filter non-excluded containers
$relevantContainers = collect($containers)->filter(function ($container) use ($excludedContainers) {
$labels = data_get($container, 'Config.Labels', []);
$serviceName = data_get($labels, 'com.docker.compose.service');
if ($serviceName && $excludedContainers->contains($serviceName)) {
continue;
}
return ! ($serviceName && $excludedContainers->contains($serviceName));
});
$relevantContainerCount++;
$containerStatus = data_get($container, 'State.Status');
$containerHealth = data_get($container, 'State.Health.Status', 'unhealthy');
if ($containerStatus === 'restarting') {
$hasRestarting = true;
$hasUnhealthy = true;
} elseif ($containerStatus === 'running') {
$hasRunning = true;
if ($containerHealth === 'unhealthy') {
$hasUnhealthy = true;
}
} elseif ($containerStatus === 'exited') {
$hasExited = true;
$hasUnhealthy = true;
}
// If all containers are excluded, calculate status from excluded containers
// but mark it with :excluded to indicate monitoring is disabled
if ($relevantContainers->isEmpty()) {
return $this->calculateExcludedStatus($containers, $excludedContainers);
}
if ($relevantContainerCount === 0) {
return 'running:healthy';
}
// Use ContainerStatusAggregator service for state machine logic
$aggregator = new ContainerStatusAggregator;
if ($hasRestarting) {
return 'degraded:unhealthy';
}
if ($hasRunning && $hasExited) {
return 'degraded:unhealthy';
}
if ($hasRunning) {
return $hasUnhealthy ? 'running:unhealthy' : 'running:healthy';
}
return 'exited:unhealthy';
return $aggregator->aggregateFromContainers($relevantContainers);
}
}
app/Jobs/PushServerUpdateJob.php
+173 -97
View File
@@ -13,6 +13,8 @@ use App\Models\Server;
use App\Models\ServiceApplication;
use App\Models\ServiceDatabase;
use App\Notifications\Container\ContainerRestarted;
use App\Services\ContainerStatusAggregator;
use App\Traits\CalculatesExcludedStatus;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
@@ -25,6 +27,7 @@ use Laravel\Horizon\Contracts\Silenced;
class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
{
use CalculatesExcludedStatus;
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $tries = 1;
@@ -67,6 +70,8 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
public Collection $applicationContainerStatuses;
public Collection $serviceContainerStatuses;
public bool $foundProxy = false;
public bool $foundLogDrainContainer = false;
@@ -90,6 +95,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
$this->foundApplicationPreviewsIds = collect();
$this->foundServiceDatabaseIds = collect();
$this->applicationContainerStatuses = collect();
$this->serviceContainerStatuses = collect();
$this->allApplicationIds = collect();
$this->allDatabaseUuids = collect();
$this->allTcpProxyUuids = collect();
@@ -99,6 +105,20 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
public function handle()
{
// Defensive initialization for Collection properties to handle queue deserialization edge cases
$this->serviceContainerStatuses ??= collect();
$this->applicationContainerStatuses ??= collect();
$this->foundApplicationIds ??= collect();
$this->foundDatabaseUuids ??= collect();
$this->foundServiceApplicationIds ??= collect();
$this->foundApplicationPreviewsIds ??= collect();
$this->foundServiceDatabaseIds ??= collect();
$this->allApplicationIds ??= collect();
$this->allDatabaseUuids ??= collect();
$this->allTcpProxyUuids ??= collect();
$this->allServiceApplicationIds ??= collect();
$this->allServiceDatabaseIds ??= collect();
// TODO: Swarm is not supported yet
if (! $this->data) {
throw new \Exception('No data provided');
@@ -108,7 +128,6 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
$this->server->sentinelHeartbeat();
$this->containers = collect(data_get($data, 'containers'));
$filesystemUsageRoot = data_get($data, 'filesystem_usage_root.used_percentage');
ServerStorageCheckJob::dispatch($this->server, $filesystemUsageRoot);
@@ -141,65 +160,88 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
foreach ($this->containers as $container) {
$containerStatus = data_get($container, 'state', 'exited');
$containerHealth = data_get($container, 'health_status', 'unhealthy');
$containerStatus = "$containerStatus ($containerHealth)";
$rawHealthStatus = data_get($container, 'health_status');
$containerHealth = $rawHealthStatus ?? 'unknown';
// Only append health status if container is not exited
if ($containerStatus !== 'exited') {
$containerStatus = "$containerStatus:$containerHealth";
}
$labels = collect(data_get($container, 'labels'));
$coolify_managed = $labels->has('coolify.managed');
if ($coolify_managed) {
$name = data_get($container, 'name');
if ($name === 'coolify-log-drain' && $this->isRunning($containerStatus)) {
$this->foundLogDrainContainer = true;
}
if ($labels->has('coolify.applicationId')) {
$applicationId = $labels->get('coolify.applicationId');
$pullRequestId = $labels->get('coolify.pullRequestId', '0');
try {
if ($pullRequestId === '0') {
if ($this->allApplicationIds->contains($applicationId) && $this->isRunning($containerStatus)) {
$this->foundApplicationIds->push($applicationId);
}
// Store container status for aggregation
if (! $this->applicationContainerStatuses->has($applicationId)) {
$this->applicationContainerStatuses->put($applicationId, collect());
}
$containerName = $labels->get('com.docker.compose.service');
if ($containerName) {
$this->applicationContainerStatuses->get($applicationId)->put($containerName, $containerStatus);
}
} else {
$previewKey = $applicationId.':'.$pullRequestId;
if ($this->allApplicationPreviewsIds->contains($previewKey) && $this->isRunning($containerStatus)) {
$this->foundApplicationPreviewsIds->push($previewKey);
}
$this->updateApplicationPreviewStatus($applicationId, $pullRequestId, $containerStatus);
if (! $coolify_managed) {
continue;
}
$name = data_get($container, 'name');
if ($name === 'coolify-log-drain' && $this->isRunning($containerStatus)) {
$this->foundLogDrainContainer = true;
}
if ($labels->has('coolify.applicationId')) {
$applicationId = $labels->get('coolify.applicationId');
$pullRequestId = $labels->get('coolify.pullRequestId', '0');
try {
if ($pullRequestId === '0') {
if ($this->allApplicationIds->contains($applicationId)) {
$this->foundApplicationIds->push($applicationId);
}
// Store container status for aggregation
if (! $this->applicationContainerStatuses->has($applicationId)) {
$this->applicationContainerStatuses->put($applicationId, collect());
}
$containerName = $labels->get('com.docker.compose.service');
if ($containerName) {
$this->applicationContainerStatuses->get($applicationId)->put($containerName, $containerStatus);
}
} catch (\Exception $e) {
}
} elseif ($labels->has('coolify.serviceId')) {
$serviceId = $labels->get('coolify.serviceId');
$subType = $labels->get('coolify.service.subType');
$subId = $labels->get('coolify.service.subId');
if ($subType === 'application' && $this->isRunning($containerStatus)) {
$this->foundServiceApplicationIds->push($subId);
$this->updateServiceSubStatus($serviceId, $subType, $subId, $containerStatus);
} elseif ($subType === 'database' && $this->isRunning($containerStatus)) {
$this->foundServiceDatabaseIds->push($subId);
$this->updateServiceSubStatus($serviceId, $subType, $subId, $containerStatus);
}
} else {
$uuid = $labels->get('com.docker.compose.service');
$type = $labels->get('coolify.type');
if ($name === 'coolify-proxy' && $this->isRunning($containerStatus)) {
$this->foundProxy = true;
} elseif ($type === 'service' && $this->isRunning($containerStatus)) {
} else {
if ($this->allDatabaseUuids->contains($uuid) && $this->isRunning($containerStatus)) {
$this->foundDatabaseUuids->push($uuid);
if ($this->allTcpProxyUuids->contains($uuid) && $this->isRunning($containerStatus)) {
$this->updateDatabaseStatus($uuid, $containerStatus, tcpProxy: true);
} else {
$this->updateDatabaseStatus($uuid, $containerStatus, tcpProxy: false);
}
$previewKey = $applicationId.':'.$pullRequestId;
if ($this->allApplicationPreviewsIds->contains($previewKey)) {
$this->foundApplicationPreviewsIds->push($previewKey);
}
$this->updateApplicationPreviewStatus($applicationId, $pullRequestId, $containerStatus);
}
} catch (\Exception $e) {
}
} elseif ($labels->has('coolify.serviceId')) {
$serviceId = $labels->get('coolify.serviceId');
$subType = $labels->get('coolify.service.subType');
$subId = $labels->get('coolify.service.subId');
if ($subType === 'application') {
$this->foundServiceApplicationIds->push($subId);
// Store container status for aggregation
$key = $serviceId.':'.$subType.':'.$subId;
if (! $this->serviceContainerStatuses->has($key)) {
$this->serviceContainerStatuses->put($key, collect());
}
$containerName = $labels->get('com.docker.compose.service');
if ($containerName) {
$this->serviceContainerStatuses->get($key)->put($containerName, $containerStatus);
}
} elseif ($subType === 'database') {
$this->foundServiceDatabaseIds->push($subId);
// Store container status for aggregation
$key = $serviceId.':'.$subType.':'.$subId;
if (! $this->serviceContainerStatuses->has($key)) {
$this->serviceContainerStatuses->put($key, collect());
}
$containerName = $labels->get('com.docker.compose.service');
if ($containerName) {
$this->serviceContainerStatuses->get($key)->put($containerName, $containerStatus);
}
}
} else {
$uuid = $labels->get('com.docker.compose.service');
$type = $labels->get('coolify.type');
if ($name === 'coolify-proxy' && $this->isRunning($containerStatus)) {
$this->foundProxy = true;
} elseif ($type === 'service' && $this->isRunning($containerStatus)) {
} else {
if ($this->allDatabaseUuids->contains($uuid) && $this->isRunning($containerStatus)) {
$this->foundDatabaseUuids->push($uuid);
if ($this->allTcpProxyUuids->contains($uuid) && $this->isRunning($containerStatus)) {
$this->updateDatabaseStatus($uuid, $containerStatus, tcpProxy: true);
} else {
$this->updateDatabaseStatus($uuid, $containerStatus, tcpProxy: false);
}
}
}
@@ -218,6 +260,9 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
// Aggregate multi-container application statuses
$this->aggregateMultiContainerStatuses();
// Aggregate multi-container service statuses
$this->aggregateServiceContainerStatuses();
$this->checkLogDrainContainer();
}
@@ -235,57 +280,28 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
// Parse docker compose to check for excluded containers
$dockerComposeRaw = data_get($application, 'docker_compose_raw');
$excludedContainers = collect();
if ($dockerComposeRaw) {
try {
$dockerCompose = \Symfony\Component\Yaml\Yaml::parse($dockerComposeRaw);
$services = data_get($dockerCompose, 'services', []);
foreach ($services as $serviceName => $serviceConfig) {
// Check if container should be excluded
$excludeFromHc = data_get($serviceConfig, 'exclude_from_hc', false);
$restartPolicy = data_get($serviceConfig, 'restart', 'always');
if ($excludeFromHc || $restartPolicy === 'no') {
$excludedContainers->push($serviceName);
}
}
} catch (\Exception $e) {
// If we can't parse, treat all containers as included
}
}
$excludedContainers = $this->getExcludedContainersFromDockerCompose($dockerComposeRaw);
// Filter out excluded containers
$relevantStatuses = $containerStatuses->filter(function ($status, $containerName) use ($excludedContainers) {
return ! $excludedContainers->contains($containerName);
});
// If all containers are excluded, don't update status
// If all containers are excluded, calculate status from excluded containers
if ($relevantStatuses->isEmpty()) {
$aggregatedStatus = $this->calculateExcludedStatusFromStrings($containerStatuses);
if ($aggregatedStatus && $application->status !== $aggregatedStatus) {
$application->status = $aggregatedStatus;
$application->save();
}
continue;
}
// Aggregate status: if any container is running, app is running
$hasRunning = false;
$hasUnhealthy = false;
foreach ($relevantStatuses as $status) {
if (str($status)->contains('running')) {
$hasRunning = true;
if (str($status)->contains('unhealthy')) {
$hasUnhealthy = true;
}
}
}
$aggregatedStatus = null;
if ($hasRunning) {
$aggregatedStatus = $hasUnhealthy ? 'running (unhealthy)' : 'running (healthy)';
} else {
// All containers are exited
$aggregatedStatus = 'exited (unhealthy)';
}
// Use ContainerStatusAggregator service for state machine logic
$aggregator = new ContainerStatusAggregator;
$aggregatedStatus = $aggregator->aggregateFromStrings($relevantStatuses, 0);
// Update application status with aggregated result
if ($aggregatedStatus && $application->status !== $aggregatedStatus) {
@@ -295,6 +311,66 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
}
}
private function aggregateServiceContainerStatuses()
{
if ($this->serviceContainerStatuses->isEmpty()) {
return;
}
foreach ($this->serviceContainerStatuses as $key => $containerStatuses) {
// Parse key: serviceId:subType:subId
[$serviceId, $subType, $subId] = explode(':', $key);
$service = $this->services->where('id', $serviceId)->first();
if (! $service) {
continue;
}
// Get the service sub-resource (ServiceApplication or ServiceDatabase)
$subResource = null;
if ($subType === 'application') {
$subResource = $service->applications()->where('id', $subId)->first();
} elseif ($subType === 'database') {
$subResource = $service->databases()->where('id', $subId)->first();
}
if (! $subResource) {
continue;
}
// Parse docker compose from service to check for excluded containers
$dockerComposeRaw = data_get($service, 'docker_compose_raw');
$excludedContainers = $this->getExcludedContainersFromDockerCompose($dockerComposeRaw);
// Filter out excluded containers
$relevantStatuses = $containerStatuses->filter(function ($status, $containerName) use ($excludedContainers) {
return ! $excludedContainers->contains($containerName);
});
// If all containers are excluded, calculate status from excluded containers
if ($relevantStatuses->isEmpty()) {
$aggregatedStatus = $this->calculateExcludedStatusFromStrings($containerStatuses);
if ($aggregatedStatus && $subResource->status !== $aggregatedStatus) {
$subResource->status = $aggregatedStatus;
$subResource->save();
}
continue;
}
// Use ContainerStatusAggregator service for state machine logic
// NOTE: Sentinel does NOT provide restart count data, so maxRestartCount is always 0
$aggregator = new ContainerStatusAggregator;
$aggregatedStatus = $aggregator->aggregateFromStrings($relevantStatuses, 0);
// Update service sub-resource status with aggregated result
if ($aggregatedStatus && $subResource->status !== $aggregatedStatus) {
$subResource->status = $aggregatedStatus;
$subResource->save();
}
}
}
private function updateApplicationStatus(string $applicationId, string $containerStatus)
{
$application = $this->applications->where('id', $applicationId)->first();
app/Jobs/ValidateAndInstallServerJob.php
+36
View File
@@ -72,6 +72,42 @@ class ValidateAndInstallServerJob implements ShouldQueue
return;
}
// Check and install prerequisites
$validationResult = $this->server->validatePrerequisites();
if (! $validationResult['success']) {
if ($this->numberOfTries >= $this->maxTries) {
$missingCommands = implode(', ', $validationResult['missing']);
$errorMessage = "Prerequisites ({$missingCommands}) could not be installed after {$this->maxTries} attempts. Please install them manually before continuing.";
$this->server->update([
'validation_logs' => $errorMessage,
'is_validating' => false,
]);
Log::error('ValidateAndInstallServer: Prerequisites installation failed after max tries', [
'server_id' => $this->server->id,
'attempts' => $this->numberOfTries,
'missing_commands' => $validationResult['missing'],
'found_commands' => $validationResult['found'],
]);
return;
}
Log::info('ValidateAndInstallServer: Installing prerequisites', [
'server_id' => $this->server->id,
'attempt' => $this->numberOfTries + 1,
'missing_commands' => $validationResult['missing'],
'found_commands' => $validationResult['found'],
]);
// Install prerequisites
$this->server->installPrerequisites();
// Retry validation after installation
self::dispatch($this->server, $this->numberOfTries + 1)->delay(now()->addSeconds(30));
return;
}
// Check if Docker is installed
$dockerInstalled = $this->server->validateDockerEngine();
$dockerComposeInstalled = $this->server->validateDockerCompose();
app/Livewire/ActivityMonitor.php
+9 -1
View File
@@ -28,12 +28,20 @@ class ActivityMonitor extends Component
protected $listeners = ['activityMonitor' => 'newMonitorActivity'];
public function newMonitorActivity($activityId, $eventToDispatch = 'activityFinished', $eventData = null)
public function newMonitorActivity($activityId, $eventToDispatch = 'activityFinished', $eventData = null, $header = null)
{
// Reset event dispatched flag for new activity
self::$eventDispatched = false;
$this->activityId = $activityId;
$this->eventToDispatch = $eventToDispatch;
$this->eventData = $eventData;
// Update header if provided
if ($header !== null) {
$this->header = $header;
}
$this->hydrateActivity();
$this->isPollingActive = true;
app/Livewire/Boarding/Index.php
+64 -1
View File
@@ -14,7 +14,10 @@ use Visus\Cuid2\Cuid2;
class Index extends Component
{
protected $listeners = ['refreshBoardingIndex' => 'validateServer'];
protected $listeners = [
'refreshBoardingIndex' => 'validateServer',
'prerequisitesInstalled' => 'handlePrerequisitesInstalled',
];
#[\Livewire\Attributes\Url(as: 'step', history: true)]
public string $currentState = 'welcome';
@@ -76,6 +79,10 @@ class Index extends Component
public ?string $minDockerVersion = null;
public int $prerequisiteInstallAttempts = 0;
public int $maxPrerequisiteInstallAttempts = 3;
public function mount()
{
if (auth()->user()?->isMember() && auth()->user()->currentTeam()->show_boarding === true) {
@@ -320,6 +327,62 @@ class Index extends Component
return handleError(error: $e, livewire: $this);
}
try {
// Check prerequisites
$validationResult = $this->createdServer->validatePrerequisites();
if (! $validationResult['success']) {
// Check if we've exceeded max attempts
if ($this->prerequisiteInstallAttempts >= $this->maxPrerequisiteInstallAttempts) {
$missingCommands = implode(', ', $validationResult['missing']);
throw new \Exception("Prerequisites ({$missingCommands}) could not be installed after {$this->maxPrerequisiteInstallAttempts} attempts. Please install them manually.");
}
// Start async installation and wait for completion via ActivityMonitor
$activity = $this->createdServer->installPrerequisites();
$this->prerequisiteInstallAttempts++;
$this->dispatch('activityMonitor', $activity->id, 'prerequisitesInstalled');
// Return early - handlePrerequisitesInstalled() will be called when installation completes
return;
}
// Prerequisites are already installed, continue with validation
$this->continueValidation();
} catch (\Throwable $e) {
return handleError(error: $e, livewire: $this);
}
}
public function handlePrerequisitesInstalled()
{
try {
// Revalidate prerequisites after installation completes
$validationResult = $this->createdServer->validatePrerequisites();
if (! $validationResult['success']) {
// Installation completed but prerequisites still missing - retry
$missingCommands = implode(', ', $validationResult['missing']);
if ($this->prerequisiteInstallAttempts >= $this->maxPrerequisiteInstallAttempts) {
throw new \Exception("Prerequisites ({$missingCommands}) could not be installed after {$this->maxPrerequisiteInstallAttempts} attempts. Please install them manually.");
}
// Try again
$activity = $this->createdServer->installPrerequisites();
$this->prerequisiteInstallAttempts++;
$this->dispatch('activityMonitor', $activity->id, 'prerequisitesInstalled');
return;
}
// Prerequisites validated successfully - continue with Docker validation
$this->continueValidation();
} catch (\Throwable $e) {
return handleError(error: $e, livewire: $this);
}
}
private function continueValidation()
{
try {
$dockerVersion = instant_remote_process(["docker version|head -2|grep -i version| awk '{print $2}'"], $this->createdServer, true);
$dockerVersion = checkMinimumDockerEngineVersion($dockerVersion);
app/Livewire/Server/ValidateAndInstall.php
+46 -1
View File
@@ -25,6 +25,8 @@ class ValidateAndInstall extends Component
public $supported_os_type = null;
public $prerequisites_installed = null;
public $docker_installed = null;
public $docker_compose_installed = null;
@@ -33,12 +35,15 @@ class ValidateAndInstall extends Component
public $error = null;
public string $installationStep = 'Prerequisites';
public bool $ask = false;
protected $listeners = [
'init',
'validateConnection',
'validateOS',
'validatePrerequisites',
'validateDockerEngine',
'validateDockerVersion',
'refresh' => '$refresh',
@@ -48,6 +53,7 @@ class ValidateAndInstall extends Component
{
$this->uptime = null;
$this->supported_os_type = null;
$this->prerequisites_installed = null;
$this->docker_installed = null;
$this->docker_version = null;
$this->docker_compose_installed = null;
@@ -69,6 +75,7 @@ class ValidateAndInstall extends Component
$this->authorize('update', $this->server);
$this->uptime = null;
$this->supported_os_type = null;
$this->prerequisites_installed = null;
$this->docker_installed = null;
$this->docker_compose_installed = null;
$this->docker_version = null;
@@ -103,6 +110,43 @@ class ValidateAndInstall extends Component
return;
}
$this->dispatch('validatePrerequisites');
}
public function validatePrerequisites()
{
$validationResult = $this->server->validatePrerequisites();
$this->prerequisites_installed = $validationResult['success'];
if (! $validationResult['success']) {
if ($this->install) {
if ($this->number_of_tries == $this->max_tries) {
$missingCommands = implode(', ', $validationResult['missing']);
$this->error = "Prerequisites ({$missingCommands}) could not be installed. Please install them manually before continuing.";
$this->server->update([
'validation_logs' => $this->error,
]);
return;
} else {
if ($this->number_of_tries <= $this->max_tries) {
$this->installationStep = 'Prerequisites';
$activity = $this->server->installPrerequisites();
$this->number_of_tries++;
$this->dispatch('activityMonitor', $activity->id, 'init', $this->number_of_tries, "{$this->installationStep} Installation Logs");
}
return;
}
} else {
$missingCommands = implode(', ', $validationResult['missing']);
$this->error = "Prerequisites ({$missingCommands}) are not installed. Please install them before continuing.";
$this->server->update([
'validation_logs' => $this->error,
]);
return;
}
}
$this->dispatch('validateDockerEngine');
}
@@ -121,9 +165,10 @@ class ValidateAndInstall extends Component
return;
} else {
if ($this->number_of_tries <= $this->max_tries) {
$this->installationStep = 'Docker';
$activity = $this->server->installDocker();
$this->number_of_tries++;
$this->dispatch('activityMonitor', $activity->id, 'init', $this->number_of_tries);
$this->dispatch('activityMonitor', $activity->id, 'init', $this->number_of_tries, "{$this->installationStep} Installation Logs");
}
return;
app/Models/Application.php
+12 -10
View File
@@ -669,21 +669,23 @@ class Application extends BaseModel
{
return Attribute::make(
get: function () {
if (! $this->relationLoaded('additional_servers') || $this->additional_servers->count() === 0) {
return $this->destination?->server?->isFunctional() ?? false;
// Check main server infrastructure health
$main_server_functional = $this->destination?->server?->isFunctional() ?? false;
if (! $main_server_functional) {
return false;
}
$additional_servers_status = $this->additional_servers->pluck('pivot.status');
$main_server_status = $this->destination?->server?->isFunctional() ?? false;
foreach ($additional_servers_status as $status) {
$server_status = str($status)->before(':')->value();
if ($server_status !== 'running') {
return false;
// Check additional servers infrastructure health (not container status!)
if ($this->relationLoaded('additional_servers') && $this->additional_servers->count() > 0) {
foreach ($this->additional_servers as $server) {
if (! $server->isFunctional()) {
return false; // Real server infrastructure problem
}
}
}
return $main_server_status;
return true;
}
);
}
app/Models/Server.php
+17
View File
@@ -4,7 +4,9 @@ namespace App\Models;
use App\Actions\Proxy\StartProxy;
use App\Actions\Server\InstallDocker;
use App\Actions\Server\InstallPrerequisites;
use App\Actions\Server\StartSentinel;
use App\Actions\Server\ValidatePrerequisites;
use App\Enums\ProxyTypes;
use App\Events\ServerReachabilityChanged;
use App\Helpers\SslHelper;
@@ -1184,6 +1186,21 @@ $schema://$host {
return InstallDocker::run($this);
}
/**
* Validate that required commands are available on the server.
*
* @return array{success: bool, missing: array<string>, found: array<string>}
*/
public function validatePrerequisites(): array
{
return ValidatePrerequisites::run($this);
}
public function installPrerequisites()
{
return InstallPrerequisites::run($this);
}
public function validateDockerEngine($throwError = false)
{
$dockerBinary = instant_remote_process(['command -v docker'], $this, false, no_sudo: true);
app/Models/Service.php
+103 -56
View File
@@ -3,6 +3,7 @@
namespace App\Models;
use App\Enums\ProcessStatus;
use App\Services\ContainerStatusAggregator;
use App\Traits\ClearsGlobalSearchCache;
use App\Traits\HasSafeStringAttribute;
use Illuminate\Database\Eloquent\Casts\Attribute;
@@ -173,6 +174,21 @@ class Service extends BaseModel
instant_remote_process(["docker network rm {$this->uuid}"], $server, false);
}
/**
* Calculate the service's aggregate status from its applications and databases.
*
* This method aggregates status from Eloquent model relationships (not Docker containers).
* It differs from the CalculatesExcludedStatus trait which works with Docker container objects
* during container inspection. This accessor runs on-demand for UI display and works with
* already-stored status strings from the database.
*
* Status format: "{status}:{health}" or "{status}:{health}:excluded"
* - Status values: running, exited, degraded, starting, paused, restarting
* - Health values: healthy, unhealthy, unknown
* - :excluded suffix: Indicates all containers are excluded from health monitoring
*
* @return string The aggregate status in format "status:health" or "status:health:excluded"
*/
public function getStatusAttribute()
{
if ($this->isStarting()) {
@@ -182,71 +198,102 @@ class Service extends BaseModel
$applications = $this->applications;
$databases = $this->databases;
$complexStatus = null;
$complexHealth = null;
[$complexStatus, $complexHealth, $hasNonExcluded] = $this->aggregateResourceStatuses(
$applications,
$databases,
excludedOnly: false
);
foreach ($applications as $application) {
if ($application->exclude_from_status) {
continue;
// If all services are excluded from status checks, calculate status from excluded containers
// but mark it with :excluded to indicate monitoring is disabled
if (! $hasNonExcluded && ($complexStatus === null && $complexHealth === null)) {
[$excludedStatus, $excludedHealth] = $this->aggregateResourceStatuses(
$applications,
$databases,
excludedOnly: true
);
// Return status with :excluded suffix to indicate monitoring is disabled
if ($excludedStatus && $excludedHealth) {
return "{$excludedStatus}:{$excludedHealth}:excluded";
}
$status = str($application->status)->before('(')->trim();
$health = str($application->status)->between('(', ')')->trim();
if ($complexStatus === 'degraded') {
continue;
}
if ($status->startsWith('running')) {
if ($complexStatus === 'exited') {
$complexStatus = 'degraded';
} else {
$complexStatus = 'running';
}
} elseif ($status->startsWith('restarting')) {
$complexStatus = 'degraded';
} elseif ($status->startsWith('exited')) {
$complexStatus = 'exited';
}
if ($health->value() === 'healthy') {
if ($complexHealth === 'unhealthy') {
continue;
}
$complexHealth = 'healthy';
} else {
$complexHealth = 'unhealthy';
// If no status was calculated at all (no containers exist), return unknown
if ($excludedStatus === null && $excludedHealth === null) {
return 'unknown:unknown:excluded';
}
return 'exited';
}
foreach ($databases as $database) {
if ($database->exclude_from_status) {
continue;
}
$status = str($database->status)->before('(')->trim();
$health = str($database->status)->between('(', ')')->trim();
if ($complexStatus === 'degraded') {
continue;
}
if ($status->startsWith('running')) {
if ($complexStatus === 'exited') {
$complexStatus = 'degraded';
} else {
$complexStatus = 'running';
}
} elseif ($status->startsWith('restarting')) {
$complexStatus = 'degraded';
} elseif ($status->startsWith('exited')) {
$complexStatus = 'exited';
}
if ($health->value() === 'healthy') {
if ($complexHealth === 'unhealthy') {
continue;
}
$complexHealth = 'healthy';
} else {
$complexHealth = 'unhealthy';
}
// If health is null/empty, return just the status without trailing colon
if ($complexHealth === null || $complexHealth === '') {
return $complexStatus;
}
return "{$complexStatus}:{$complexHealth}";
}
/**
* Aggregate status and health from collections of applications and databases.
*
* This helper method consolidates status aggregation logic using ContainerStatusAggregator.
* It processes container status strings stored in the database (not live Docker data).
*
* @param \Illuminate\Database\Eloquent\Collection $applications Collection of Application models
* @param \Illuminate\Database\Eloquent\Collection $databases Collection of Database models
* @param bool $excludedOnly If true, only process excluded containers; if false, only process non-excluded
* @return array{0: string|null, 1: string|null, 2?: bool} [status, health, hasNonExcluded (only when excludedOnly=false)]
*/
private function aggregateResourceStatuses($applications, $databases, bool $excludedOnly = false): array
{
$hasNonExcluded = false;
$statusStrings = collect();
// Process both applications and databases using the same logic
$resources = $applications->concat($databases);
foreach ($resources as $resource) {
$isExcluded = $resource->exclude_from_status || str($resource->status)->contains(':excluded');
// Filter based on excludedOnly flag
if ($excludedOnly && ! $isExcluded) {
continue;
}
if (! $excludedOnly && $isExcluded) {
continue;
}
if (! $excludedOnly) {
$hasNonExcluded = true;
}
// Strip :excluded suffix before aggregation (it's in the 3rd part of "status:health:excluded")
$status = str($resource->status)->before(':excluded')->toString();
$statusStrings->push($status);
}
// If no status strings collected, return nulls
if ($statusStrings->isEmpty()) {
return $excludedOnly ? [null, null] : [null, null, $hasNonExcluded];
}
// Use ContainerStatusAggregator service for state machine logic
$aggregator = new ContainerStatusAggregator;
$aggregatedStatus = $aggregator->aggregateFromStrings($statusStrings);
// Parse the aggregated "status:health" string
$parts = explode(':', $aggregatedStatus);
$status = $parts[0] ?? null;
$health = $parts[1] ?? null;
if ($excludedOnly) {
return [$status, $health];
}
return [$status, $health, $hasNonExcluded];
}
public function extraFields()
{
$fields = collect([]);
app/Models/ServiceApplication.php
+50 -49
View File
@@ -189,65 +189,66 @@ class ServiceApplication extends BaseModel
public function getRequiredPort(): ?int
{
try {
// Normalize container name same way as variable creation
// (uppercase, replace - and . with _)
$normalizedName = str($this->name)
->upper()
->replace('-', '_')
->replace('.', '_')
->value();
// Get all environment variables from the service
$serviceEnvVars = $this->service->environment_variables()->get();
// Parse the Docker Compose to find SERVICE_URL/SERVICE_FQDN variables DIRECTLY DECLARED
// for this specific service container (not just referenced from other containers)
$dockerComposeRaw = data_get($this->service, 'docker_compose_raw');
if (! $dockerComposeRaw) {
// Fall back to service-level port if no compose file
return $this->service->getRequiredPort();
}
// Look for SERVICE_FQDN_* or SERVICE_URL_* variables that match this container
foreach ($serviceEnvVars as $envVar) {
$key = str($envVar->key);
$dockerCompose = \Symfony\Component\Yaml\Yaml::parse($dockerComposeRaw);
$serviceConfig = data_get($dockerCompose, "services.{$this->name}");
if (! $serviceConfig) {
return $this->service->getRequiredPort();
}
// Check if this is a SERVICE_FQDN_* or SERVICE_URL_* variable
if (! $key->startsWith('SERVICE_FQDN_') && ! $key->startsWith('SERVICE_URL_')) {
continue;
}
// Extract the part after SERVICE_FQDN_ or SERVICE_URL_
if ($key->startsWith('SERVICE_FQDN_')) {
$suffix = $key->after('SERVICE_FQDN_');
} else {
$suffix = $key->after('SERVICE_URL_');
}
$environment = data_get($serviceConfig, 'environment', []);
// Check if this variable starts with our normalized container name
// Format: {NORMALIZED_NAME}_{PORT} or just {NORMALIZED_NAME}
if (! $suffix->startsWith($normalizedName)) {
\Log::debug('[ServiceApplication::getRequiredPort] Suffix does not match container', [
'expected_start' => $normalizedName,
'actual_suffix' => $suffix->value(),
]);
// Extract SERVICE_URL and SERVICE_FQDN variables DIRECTLY DECLARED in this service's environment
// (not variables that are merely referenced with ${VAR} syntax)
$portFound = null;
foreach ($environment as $key => $value) {
if (is_int($key) && is_string($value)) {
// List-style: "- SERVICE_URL_APP_3000" or "- SERVICE_URL_APP_3000=value"
// Extract variable name (before '=' if present)
$envVarName = str($value)->before('=')->trim();
continue;
}
// Only process direct declarations
if ($envVarName->startsWith('SERVICE_FQDN_') || $envVarName->startsWith('SERVICE_URL_')) {
// Parse to check if it has a port suffix
$parsed = parseServiceEnvironmentVariable($envVarName->value());
if ($parsed['has_port'] && $parsed['port']) {
// Found a port-specific variable for this service
$portFound = (int) $parsed['port'];
break;
}
}
} elseif (is_string($key)) {
// Map-style: "SERVICE_URL_APP_3000: value" or "SERVICE_FQDN_DB: localhost"
$envVarName = str($key);
// Check if there's a port suffix after the container name
// The suffix should be exactly NORMALIZED_NAME or NORMALIZED_NAME_PORT
$afterName = $suffix->after($normalizedName)->value();
// If there's content after the name, it should start with underscore
if ($afterName !== '' && str($afterName)->startsWith('_')) {
// Extract port: _3210 -> 3210
$port = str($afterName)->after('_')->value();
// Validate that the extracted port is numeric
if (is_numeric($port)) {
\Log::debug('[ServiceApplication::getRequiredPort] MATCH FOUND - Returning port', [
'port' => (int) $port,
]);
return (int) $port;
// Only process direct declarations
if ($envVarName->startsWith('SERVICE_FQDN_') || $envVarName->startsWith('SERVICE_URL_')) {
// Parse to check if it has a port suffix
$parsed = parseServiceEnvironmentVariable($envVarName->value());
if ($parsed['has_port'] && $parsed['port']) {
// Found a port-specific variable for this service
$portFound = (int) $parsed['port'];
break;
}
}
}
}
// Fall back to service-level port if no port-specific variable is found
$fallbackPort = $this->service->getRequiredPort();
// If a port was found in the template, return it
if ($portFound !== null) {
return $portFound;
}
return $fallbackPort;
// No port-specific variables found for this service, return null
// (DO NOT fall back to service-level port, as that applies to all services)
return null;
} catch (\Throwable $e) {
return null;
}
app/Services/ContainerStatusAggregator.php
+251
View File
@@ -0,0 +1,251 @@
<?php
namespace App\Services;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Log;
/**
* Container Status Aggregator Service
*
* Centralized service for aggregating container statuses into a single status string.
* Uses a priority-based state machine to determine the overall status from multiple containers.
*
* Output Format: Colon-separated (e.g., "running:healthy", "degraded:unhealthy")
* This format is used throughout the backend for consistency and machine-readability.
* UI components transform this to human-readable format (e.g., "Running (Healthy)").
*
* State Priority (highest to lowest):
* 1. Restarting → degraded:unhealthy
* 2. Crash Loop (exited with restarts) → degraded:unhealthy
* 3. Mixed (running + exited) → degraded:unhealthy
* 4. Running → running:healthy/unhealthy/unknown
* 5. Dead/Removing → degraded:unhealthy
* 6. Paused → paused:unknown
* 7. Starting/Created → starting:unknown
* 8. Exited → exited
*/
class ContainerStatusAggregator
{
/**
* Aggregate container statuses from status strings into a single status.
*
* @param Collection $containerStatuses Collection of status strings (e.g., "running (healthy)", "running:healthy")
* @param int $maxRestartCount Maximum restart count across containers (for crash loop detection)
* @return string Aggregated status in colon format (e.g., "running:healthy")
*/
public function aggregateFromStrings(Collection $containerStatuses, int $maxRestartCount = 0): string
{
// Validate maxRestartCount parameter
if ($maxRestartCount < 0) {
Log::warning('Negative maxRestartCount corrected to 0', [
'original_value' => $maxRestartCount,
]);
$maxRestartCount = 0;
}
if ($maxRestartCount > 1000) {
Log::warning('High maxRestartCount detected', [
'maxRestartCount' => $maxRestartCount,
'containers' => $containerStatuses->count(),
]);
}
if ($containerStatuses->isEmpty()) {
return 'exited';
}
// Initialize state flags
$hasRunning = false;
$hasRestarting = false;
$hasUnhealthy = false;
$hasUnknown = false;
$hasExited = false;
$hasStarting = false;
$hasPaused = false;
$hasDead = false;
// Parse each status string and set flags
foreach ($containerStatuses as $status) {
if (str($status)->contains('restarting')) {
$hasRestarting = true;
} elseif (str($status)->contains('running')) {
$hasRunning = true;
if (str($status)->contains('unhealthy')) {
$hasUnhealthy = true;
}
if (str($status)->contains('unknown')) {
$hasUnknown = true;
}
} elseif (str($status)->contains('exited')) {
$hasExited = true;
} elseif (str($status)->contains('created') || str($status)->contains('starting')) {
$hasStarting = true;
} elseif (str($status)->contains('paused')) {
$hasPaused = true;
} elseif (str($status)->contains('dead') || str($status)->contains('removing')) {
$hasDead = true;
}
}
// Priority-based status resolution
return $this->resolveStatus(
$hasRunning,
$hasRestarting,
$hasUnhealthy,
$hasUnknown,
$hasExited,
$hasStarting,
$hasPaused,
$hasDead,
$maxRestartCount
);
}
/**
* Aggregate container statuses from Docker container objects.
*
* @param Collection $containers Collection of Docker container objects with State property
* @param int $maxRestartCount Maximum restart count across containers (for crash loop detection)
* @return string Aggregated status in colon format (e.g., "running:healthy")
*/
public function aggregateFromContainers(Collection $containers, int $maxRestartCount = 0): string
{
// Validate maxRestartCount parameter
if ($maxRestartCount < 0) {
Log::warning('Negative maxRestartCount corrected to 0', [
'original_value' => $maxRestartCount,
]);
$maxRestartCount = 0;
}
if ($maxRestartCount > 1000) {
Log::warning('High maxRestartCount detected', [
'maxRestartCount' => $maxRestartCount,
'containers' => $containers->count(),
]);
}
if ($containers->isEmpty()) {
return 'exited';
}
// Initialize state flags
$hasRunning = false;
$hasRestarting = false;
$hasUnhealthy = false;
$hasUnknown = false;
$hasExited = false;
$hasStarting = false;
$hasPaused = false;
$hasDead = false;
// Parse each container object and set flags
foreach ($containers as $container) {
$state = data_get($container, 'State.Status', 'exited');
$health = data_get($container, 'State.Health.Status');
if ($state === 'restarting') {
$hasRestarting = true;
} elseif ($state === 'running') {
$hasRunning = true;
if ($health === 'unhealthy') {
$hasUnhealthy = true;
} elseif (is_null($health) || $health === 'starting') {
$hasUnknown = true;
}
} elseif ($state === 'exited') {
$hasExited = true;
} elseif ($state === 'created' || $state === 'starting') {
$hasStarting = true;
} elseif ($state === 'paused') {
$hasPaused = true;
} elseif ($state === 'dead' || $state === 'removing') {
$hasDead = true;
}
}
// Priority-based status resolution
return $this->resolveStatus(
$hasRunning,
$hasRestarting,
$hasUnhealthy,
$hasUnknown,
$hasExited,
$hasStarting,
$hasPaused,
$hasDead,
$maxRestartCount
);
}
/**
* Resolve the aggregated status based on state flags (priority-based state machine).
*
* @param bool $hasRunning Has at least one running container
* @param bool $hasRestarting Has at least one restarting container
* @param bool $hasUnhealthy Has at least one unhealthy container
* @param bool $hasUnknown Has at least one container with unknown health
* @param bool $hasExited Has at least one exited container
* @param bool $hasStarting Has at least one starting/created container
* @param bool $hasPaused Has at least one paused container
* @param bool $hasDead Has at least one dead/removing container
* @param int $maxRestartCount Maximum restart count (for crash loop detection)
* @return string Status in colon format (e.g., "running:healthy")
*/
private function resolveStatus(
bool $hasRunning,
bool $hasRestarting,
bool $hasUnhealthy,
bool $hasUnknown,
bool $hasExited,
bool $hasStarting,
bool $hasPaused,
bool $hasDead,
int $maxRestartCount
): string {
// Priority 1: Restarting containers (degraded state)
if ($hasRestarting) {
return 'degraded:unhealthy';
}
// Priority 2: Crash loop detection (exited with restart count > 0)
if ($hasExited && $maxRestartCount > 0) {
return 'degraded:unhealthy';
}
// Priority 3: Mixed state (some running, some exited = degraded)
if ($hasRunning && $hasExited) {
return 'degraded:unhealthy';
}
// Priority 4: Running containers (check health status)
if ($hasRunning) {
if ($hasUnhealthy) {
return 'running:unhealthy';
} elseif ($hasUnknown) {
return 'running:unknown';
} else {
return 'running:healthy';
}
}
// Priority 5: Dead or removing containers
if ($hasDead) {
return 'degraded:unhealthy';
}
// Priority 6: Paused containers
if ($hasPaused) {
return 'paused:unknown';
}
// Priority 7: Starting/created containers
if ($hasStarting) {
return 'starting:unknown';
}
// Priority 8: All containers exited (no restart count = truly stopped)
return 'exited';
}
}
app/Traits/CalculatesExcludedStatus.php
+166
View File
@@ -0,0 +1,166 @@
<?php
namespace App\Traits;
use App\Services\ContainerStatusAggregator;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Log;
use Symfony\Component\Yaml\Exception\ParseException;
trait CalculatesExcludedStatus
{
/**
* Calculate status for containers when all containers are excluded from health checks.
*
* This method processes excluded containers and returns a status with :excluded suffix
* to indicate that monitoring is disabled but still show the actual container state.
*
* @param Collection $containers Collection of container objects from Docker inspect
* @param Collection $excludedContainers Collection of container names that are excluded
* @return string Status string with :excluded suffix (e.g., 'running:unhealthy:excluded')
*/
protected function calculateExcludedStatus(Collection $containers, Collection $excludedContainers): string
{
// Filter to only excluded containers
$excludedOnly = $containers->filter(function ($container) use ($excludedContainers) {
$labels = data_get($container, 'Config.Labels', []);
$serviceName = data_get($labels, 'com.docker.compose.service');
return $serviceName && $excludedContainers->contains($serviceName);
});
// Use ContainerStatusAggregator service for state machine logic
$aggregator = new ContainerStatusAggregator;
$status = $aggregator->aggregateFromContainers($excludedOnly);
// Append :excluded suffix
return $this->appendExcludedSuffix($status);
}
/**
* Calculate status for containers when all containers are excluded (simplified version).
*
* This version works with status strings (e.g., "running:healthy") instead of full
* container objects, suitable for Sentinel updates that don't have full container data.
*
* @param Collection $containerStatuses Collection of status strings keyed by container name
* @return string Status string with :excluded suffix
*/
protected function calculateExcludedStatusFromStrings(Collection $containerStatuses): string
{
// Use ContainerStatusAggregator service for state machine logic
$aggregator = new ContainerStatusAggregator;
$status = $aggregator->aggregateFromStrings($containerStatuses);
// Append :excluded suffix
$finalStatus = $this->appendExcludedSuffix($status);
return $finalStatus;
}
/**
* Append :excluded suffix to a status string.
*
* Converts status formats like:
* - "running:healthy" → "running:healthy:excluded"
* - "degraded:unhealthy" → "degraded:excluded" (simplified)
* - "paused:unknown" → "paused:excluded" (simplified)
*
* @param string $status The base status string
* @return string Status with :excluded suffix
*/
private function appendExcludedSuffix(string $status): string
{
// For degraded states, simplify to just "degraded:excluded"
if (str($status)->startsWith('degraded')) {
return 'degraded:excluded';
}
// For paused/starting/exited states, simplify to just "state:excluded"
if (str($status)->startsWith('paused')) {
return 'paused:excluded';
}
if (str($status)->startsWith('starting')) {
return 'starting:excluded';
}
if (str($status)->startsWith('exited')) {
return 'exited';
}
// For running states, keep the health status: "running:healthy:excluded"
return "$status:excluded";
}
/**
* Get excluded containers from docker-compose YAML.
*
* Containers are excluded if:
* - They have exclude_from_hc: true label
* - They have restart: no policy
*
* @param string|null $dockerComposeRaw The raw docker-compose YAML content
* @return Collection Collection of excluded container names
*/
protected function getExcludedContainersFromDockerCompose(?string $dockerComposeRaw): Collection
{
$excludedContainers = collect();
if (! $dockerComposeRaw) {
return $excludedContainers;
}
try {
$dockerCompose = \Symfony\Component\Yaml\Yaml::parse($dockerComposeRaw);
// Validate structure
if (! is_array($dockerCompose)) {
Log::warning('Docker Compose YAML did not parse to array', [
'yaml_length' => strlen($dockerComposeRaw),
'parsed_type' => gettype($dockerCompose),
]);
return $excludedContainers;
}
$services = data_get($dockerCompose, 'services', []);
if (! is_array($services)) {
Log::warning('Docker Compose services is not an array', [
'services_type' => gettype($services),
]);
return $excludedContainers;
}
foreach ($services as $serviceName => $serviceConfig) {
$excludeFromHc = data_get($serviceConfig, 'exclude_from_hc', false);
$restartPolicy = data_get($serviceConfig, 'restart', 'always');
if ($excludeFromHc || $restartPolicy === 'no') {
$excludedContainers->push($serviceName);
}
}
} catch (ParseException $e) {
// Specific YAML parsing errors
Log::warning('Failed to parse Docker Compose YAML for health check exclusions', [
'error' => $e->getMessage(),
'line' => $e->getParsedLine(),
'snippet' => $e->getSnippet(),
]);
return $excludedContainers;
} catch (\Exception $e) {
// Unexpected errors
Log::error('Unexpected error parsing Docker Compose YAML', [
'error' => $e->getMessage(),
'trace' => $e->getTraceAsString(),
]);
return $excludedContainers;
}
return $excludedContainers;
}
}