fix(server): reset disk alerts only after usage recovers

Require usage to fall five percentage points below the threshold before releasing the notification throttle. Reset it after Sentinel reports recovery so a new spike alerts immediately.

Add regression coverage for recovery and usage fluctuating near the threshold.

app/Jobs/PushServerUpdateJob.php

@@ -14,6 +14,7 @@ use App\Actions\Shared\ComplexStatusCheck;
 use App\Events\ServiceChecked;
 use App\Models\Application;
 use App\Models\ApplicationPreview;
+use App\Models\NotificationThrottle;
 use App\Models\Server;
 use App\Models\ServiceApplication;
 use App\Models\ServiceDatabase;
@@ -30,6 +31,7 @@ use App\Models\StandaloneSqlite;
 use App\Models\SwarmDocker;
 use App\Notifications\Application\RestartLimitReached as ApplicationRestartLimitReached;
 use App\Notifications\Container\ContainerRestarted;
+use App\Notifications\Server\HighDiskUsage;
 use App\Services\ContainerStatusAggregator;
 use App\Services\RestartCountTracker;
 use App\Traits\CalculatesExcludedStatus;
@@ -197,14 +199,22 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
         // it is wasted work — and most servers sit well below the threshold.
         $diskThreshold = data_get($this->server, 'settings.server_disk_usage_notification_threshold', 80);
         $storageCacheKey = 'storage-check:'.$this->server->id;
+        // Set while usage was high, so the throttle is released only once when usage recovers.
+        $highUsageCacheKey = 'storage-high:'.$this->server->id;
         $lastPercentage = Cache::get($storageCacheKey);
         if ($filesystemUsageRoot !== null
             && $filesystemUsageRoot >= $diskThreshold
             && (string) $lastPercentage !== (string) $filesystemUsageRoot) {
             Cache::put($storageCacheKey, $filesystemUsageRoot, 600);
+            Cache::forever($highUsageCacheKey, true);
             ServerStorageCheckJob::dispatch($this->server, $filesystemUsageRoot);
         } elseif ($filesystemUsageRoot !== null && $filesystemUsageRoot < $diskThreshold) {
             Cache::forget($storageCacheKey);
+            // The storage check does not run below the threshold, so the next spike alerts again only
+            // when the throttle is released here.
+            if (HighDiskUsage::hasRecovered($filesystemUsageRoot, $diskThreshold) && Cache::pull($highUsageCacheKey)) {
+                NotificationThrottle::release($this->server, HighDiskUsage::class);
+            }
         }
 
         if ($this->containers->isEmpty() && ! $this->isCompleteSnapshot()) {

app/Jobs/ServerStorageCheckJob.php

@@ -60,7 +60,7 @@ class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue, Silenced
             }
             if ($this->percentage > $serverDiskUsageNotificationThreshold) {
                 $team->notify(new HighDiskUsage($this->server, $this->percentage, $serverDiskUsageNotificationThreshold));
-            } else {
+            } elseif (HighDiskUsage::hasRecovered($this->percentage, $serverDiskUsageNotificationThreshold)) {
                 // Usage recovered: the next spike should alert again instead of waiting for the interval.
                 NotificationThrottle::release($this->server, HighDiskUsage::class);
             }

app/Notifications/Server/HighDiskUsage.php

@@ -13,6 +13,17 @@ use Illuminate\Notifications\Messages\MailMessage;
 
 class HighDiskUsage extends CustomEmailNotification implements ThrottledNotification
 {
+    /**
+     * Percentage points below the threshold that disk usage must reach before a new spike alerts
+     * again, so usage that moves around the threshold does not send an alert on every change.
+     */
+    public const RECOVERY_MARGIN = 5;
+
+    public static function hasRecovered(int|float $diskUsage, int|float $threshold): bool
+    {
+        return $diskUsage <= $threshold - self::RECOVERY_MARGIN;
+    }
+
     public function __construct(public Server $server, public int $disk_usage, public int $server_disk_usage_notification_threshold)
     {
         $this->onQueue('high');

tests/Feature/Server/ServerStorageCheckNotificationIntervalTest.php

@@ -1,5 +1,6 @@
 <?php
 
+use App\Jobs\PushServerUpdateJob;
 use App\Jobs\ServerStorageCheckJob;
 use App\Livewire\Server\Advanced;
 use App\Models\InstanceSettings;
@@ -139,3 +140,45 @@ it('does not let a member change the notification interval', function () {
 
     expect($server->settings->fresh()->server_disk_usage_notification_interval_hours)->toBe(24);
 });
+
+function pushDiskUsage(Server $server, int $percentage): void
+{
+    (new PushServerUpdateJob($server->fresh(), [
+        'containers' => [],
+        'filesystem_usage_root' => ['used_percentage' => $percentage],
+    ]))->handle();
+}
+
+it('notifies again on a new spike after disk usage recovered on a Sentinel server', function () {
+    $server = diskUsageIntervalServer(intervalHours: 24);
+
+    pushDiskUsage($server, 93);
+    Carbon::setTestNow(now()->addHour());
+    pushDiskUsage($server, 40);
+    Carbon::setTestNow(now()->addHour());
+    pushDiskUsage($server, 92);
+
+    Notification::assertSentToTimes($server->team, HighDiskUsage::class, 2);
+});
+
+it('does not notify again while Sentinel disk usage moves around the threshold', function () {
+    $server = diskUsageIntervalServer(intervalHours: 24);
+
+    foreach ([81, 80, 81, 80, 79, 81, 78, 82] as $percentage) {
+        pushDiskUsage($server, $percentage);
+        Carbon::setTestNow(now()->addMinutes(2));
+    }
+
+    Notification::assertSentToTimes($server->team, HighDiskUsage::class, 1);
+});
+
+it('does not notify again while checked disk usage moves around the threshold', function () {
+    $server = diskUsageIntervalServer(intervalHours: 24);
+
+    foreach ([81, 80, 81, 76, 82] as $percentage) {
+        runDiskUsageCheck($server, $percentage);
+        Carbon::setTestNow(now()->addMinutes(2));
+    }
+
+    Notification::assertSentToTimes($server->team, HighDiskUsage::class, 1);
+});