fix(sentinel): refresh UI on container health and status changes

Include health status in push deduplication and broadcast ServiceChecked after processing complete or partial snapshots.
This commit is contained in:
Andras Bacsai
2026-10-04 16:25:00 +02:00
parent e837a4f0b4
commit f63bf5abaa
5 changed files with 79 additions and 10 deletions
@@ -179,14 +179,14 @@ class SentinelController extends Controller
/**
* Build a stable hash of container state.
*
* Covers [name, state, restart_count] only — metrics, filesystem_usage_root, and
* health_status are excluded on purpose. Disk % churns constantly, and
* health checks can flap between starting/healthy/unhealthy while the
* container lifecycle state remains unchanged. Both would otherwise defeat
* the hash and dispatch DB-heavy PushServerUpdateJob instances too often.
* The snapshot completeness flag is included so a complete snapshot always
* dispatches after a partial snapshot. Sorted by name so container ordering
* from Sentinel does not affect the hash.
* Covers [name, state, health_status, restart_count] only — metrics and
* filesystem_usage_root are excluded on purpose because disk % churns
* constantly and would dispatch PushServerUpdateJob on every push. Health is
* included so a healthy/unhealthy change reaches the UI on the next push
* instead of after the force window. The snapshot completeness flag is
* included so a complete snapshot always dispatches after a partial
* snapshot. Sorted by name so container ordering from Sentinel does not
* affect the hash.
*/
private function containerStateHash(array $data): string
{
@@ -194,6 +194,7 @@ class SentinelController extends Controller
->map(fn ($c) => [
'name' => data_get($c, 'name'),
'state' => data_get($c, 'state'),
'health_status' => data_get($c, 'health_status'),
'restart_count' => data_get($c, 'restart_count'),
])
->sortBy('name')
+5
View File
@@ -11,6 +11,7 @@ use App\Actions\Proxy\StartProxy;
use App\Actions\Server\StartLogDrain;
use App\Actions\Service\StopServiceApplication;
use App\Actions\Shared\ComplexStatusCheck;
use App\Events\ServiceChecked;
use App\Models\Application;
use App\Models\ApplicationPreview;
use App\Models\Server;
@@ -366,6 +367,8 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
}
if (! $this->isCompleteSnapshot()) {
ServiceChecked::dispatch($this->server->team_id);
return;
}
@@ -390,6 +393,8 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
$this->aggregateServiceContainerStatuses();
$this->checkLogDrainContainer();
ServiceChecked::dispatch($this->server->team_id);
}
private function isCompleteSnapshot(): bool
@@ -1,5 +1,6 @@
<?php
use App\Events\ServiceChecked;
use App\Jobs\PushServerUpdateJob;
use App\Models\Environment;
use App\Models\Project;
@@ -8,6 +9,7 @@ use App\Models\StandaloneDocker;
use App\Models\StandalonePostgresql;
use App\Models\Team;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\Event;
uses(RefreshDatabase::class);
@@ -224,6 +226,7 @@ test('partial sentinel snapshots do not trigger missing tcp proxy recovery', fun
$server = $database->destination->server;
Queue::fake();
Event::fake([ServiceChecked::class]);
$data = [
'snapshot' => [
@@ -252,3 +255,59 @@ test('partial sentinel snapshots do not trigger missing tcp proxy recovery', fun
expect($database->status)->toBe('running:healthy');
Queue::assertNothingPushed();
});
test('sentinel push broadcasts ServiceChecked to the server team after updating statuses', function (bool $complete) {
$team = Team::factory()->create();
$database = createPushUpdatePostgresql($team, [
'status' => 'exited',
]);
$server = $database->destination->server;
Event::fake([ServiceChecked::class]);
$data = [
'snapshot' => [
'version' => 1,
'complete' => $complete,
],
'containers' => [
[
'name' => $database->uuid,
'state' => 'running',
'health_status' => 'healthy',
'labels' => [
'coolify.managed' => 'true',
'coolify.type' => 'database',
'com.docker.compose.service' => $database->uuid,
],
],
],
];
$job = new PushServerUpdateJob($server, $data);
$job->handle();
expect($database->refresh()->status)->toBe('running:healthy');
Event::assertDispatched(ServiceChecked::class, fn (ServiceChecked $event) => $event->teamId === $team->id);
})->with([
'complete snapshot' => true,
'partial snapshot' => false,
]);
test('sentinel push does not broadcast ServiceChecked for an empty partial snapshot', function () {
$team = Team::factory()->create();
$database = createPushUpdatePostgresql($team, [
'status' => 'running:healthy',
]);
$server = $database->destination->server;
Event::fake([ServiceChecked::class]);
$job = new PushServerUpdateJob($server, [
'snapshot' => ['version' => 1, 'complete' => false],
'containers' => [],
]);
$job->handle();
Event::assertNotDispatched(ServiceChecked::class);
});
@@ -1,11 +1,13 @@
<?php
use App\Events\ServiceChecked;
use App\Jobs\PushServerUpdateJob;
use App\Jobs\ServerStorageCheckJob;
use App\Models\Server;
use App\Models\Team;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Event;
use Illuminate\Support\Facades\Queue;
uses(RefreshDatabase::class);
@@ -137,6 +139,8 @@ it('does not dispatch proxy network work when the proxy is running', function ()
'filesystem_usage_root' => ['used_percentage' => 10],
];
Event::fake([ServiceChecked::class]);
(new PushServerUpdateJob($server, $data))->handle();
Queue::assertNothingPushed();
@@ -175,14 +175,14 @@ it('dispatches the job when only the container restart count changes', function
Queue::assertPushed(PushServerUpdateJob::class, 2);
});
it('ignores health status changes while container lifecycle state is unchanged', function () {
it('dispatches the job when only the container health status changes', function () {
$healthy = [['name' => 'app-1', 'state' => 'running', 'health_status' => 'healthy']];
$unhealthy = [['name' => 'app-1', 'state' => 'running', 'health_status' => 'unhealthy']];
pushSentinel($this->token, sentinelPayload($healthy))->assertOk();
pushSentinel($this->token, sentinelPayload($unhealthy))->assertOk();
Queue::assertPushed(PushServerUpdateJob::class, 1);
Queue::assertPushed(PushServerUpdateJob::class, 2);
});
it('ignores disk percentage changes (excluded from the hash)', function () use ($running) {