diff --git a/app/Jobs/PushServerUpdateJob.php b/app/Jobs/PushServerUpdateJob.php index b6b358512a..768ebddb22 100644 --- a/app/Jobs/PushServerUpdateJob.php +++ b/app/Jobs/PushServerUpdateJob.php @@ -14,6 +14,7 @@ use App\Actions\Shared\ComplexStatusCheck; use App\Events\ServiceChecked; use App\Models\Application; use App\Models\ApplicationPreview; +use App\Models\NotificationThrottle; use App\Models\Server; use App\Models\ServiceApplication; use App\Models\ServiceDatabase; @@ -30,6 +31,7 @@ use App\Models\StandaloneSqlite; use App\Models\SwarmDocker; use App\Notifications\Application\RestartLimitReached as ApplicationRestartLimitReached; use App\Notifications\Container\ContainerRestarted; +use App\Notifications\Server\HighDiskUsage; use App\Services\ContainerStatusAggregator; use App\Services\RestartCountTracker; use App\Traits\CalculatesExcludedStatus; @@ -197,14 +199,22 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced // it is wasted work — and most servers sit well below the threshold. $diskThreshold = data_get($this->server, 'settings.server_disk_usage_notification_threshold', 80); $storageCacheKey = 'storage-check:'.$this->server->id; + // Set while usage was high, so the throttle is released only once when usage recovers. + $highUsageCacheKey = 'storage-high:'.$this->server->id; $lastPercentage = Cache::get($storageCacheKey); if ($filesystemUsageRoot !== null && $filesystemUsageRoot >= $diskThreshold && (string) $lastPercentage !== (string) $filesystemUsageRoot) { Cache::put($storageCacheKey, $filesystemUsageRoot, 600); + Cache::forever($highUsageCacheKey, true); ServerStorageCheckJob::dispatch($this->server, $filesystemUsageRoot); } elseif ($filesystemUsageRoot !== null && $filesystemUsageRoot < $diskThreshold) { Cache::forget($storageCacheKey); + // The storage check does not run below the threshold, so the next spike alerts again only + // when the throttle is released here. + if (HighDiskUsage::hasRecovered($filesystemUsageRoot, $diskThreshold) && Cache::pull($highUsageCacheKey)) { + NotificationThrottle::release($this->server, HighDiskUsage::class); + } } if ($this->containers->isEmpty() && ! $this->isCompleteSnapshot()) { diff --git a/app/Jobs/ServerStorageCheckJob.php b/app/Jobs/ServerStorageCheckJob.php index 1e1b2c3c2a..f8bac23516 100644 --- a/app/Jobs/ServerStorageCheckJob.php +++ b/app/Jobs/ServerStorageCheckJob.php @@ -60,7 +60,7 @@ class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue, Silenced } if ($this->percentage > $serverDiskUsageNotificationThreshold) { $team->notify(new HighDiskUsage($this->server, $this->percentage, $serverDiskUsageNotificationThreshold)); - } else { + } elseif (HighDiskUsage::hasRecovered($this->percentage, $serverDiskUsageNotificationThreshold)) { // Usage recovered: the next spike should alert again instead of waiting for the interval. NotificationThrottle::release($this->server, HighDiskUsage::class); } diff --git a/app/Notifications/Server/HighDiskUsage.php b/app/Notifications/Server/HighDiskUsage.php index be0517b759..acf938427b 100644 --- a/app/Notifications/Server/HighDiskUsage.php +++ b/app/Notifications/Server/HighDiskUsage.php @@ -13,6 +13,17 @@ use Illuminate\Notifications\Messages\MailMessage; class HighDiskUsage extends CustomEmailNotification implements ThrottledNotification { + /** + * Percentage points below the threshold that disk usage must reach before a new spike alerts + * again, so usage that moves around the threshold does not send an alert on every change. + */ + public const RECOVERY_MARGIN = 5; + + public static function hasRecovered(int|float $diskUsage, int|float $threshold): bool + { + return $diskUsage <= $threshold - self::RECOVERY_MARGIN; + } + public function __construct(public Server $server, public int $disk_usage, public int $server_disk_usage_notification_threshold) { $this->onQueue('high'); diff --git a/tests/Feature/Server/ServerStorageCheckNotificationIntervalTest.php b/tests/Feature/Server/ServerStorageCheckNotificationIntervalTest.php index d4e9563396..58e704c32f 100644 --- a/tests/Feature/Server/ServerStorageCheckNotificationIntervalTest.php +++ b/tests/Feature/Server/ServerStorageCheckNotificationIntervalTest.php @@ -1,5 +1,6 @@ settings->fresh()->server_disk_usage_notification_interval_hours)->toBe(24); }); + +function pushDiskUsage(Server $server, int $percentage): void +{ + (new PushServerUpdateJob($server->fresh(), [ + 'containers' => [], + 'filesystem_usage_root' => ['used_percentage' => $percentage], + ]))->handle(); +} + +it('notifies again on a new spike after disk usage recovered on a Sentinel server', function () { + $server = diskUsageIntervalServer(intervalHours: 24); + + pushDiskUsage($server, 93); + Carbon::setTestNow(now()->addHour()); + pushDiskUsage($server, 40); + Carbon::setTestNow(now()->addHour()); + pushDiskUsage($server, 92); + + Notification::assertSentToTimes($server->team, HighDiskUsage::class, 2); +}); + +it('does not notify again while Sentinel disk usage moves around the threshold', function () { + $server = diskUsageIntervalServer(intervalHours: 24); + + foreach ([81, 80, 81, 80, 79, 81, 78, 82] as $percentage) { + pushDiskUsage($server, $percentage); + Carbon::setTestNow(now()->addMinutes(2)); + } + + Notification::assertSentToTimes($server->team, HighDiskUsage::class, 1); +}); + +it('does not notify again while checked disk usage moves around the threshold', function () { + $server = diskUsageIntervalServer(intervalHours: 24); + + foreach ([81, 80, 81, 76, 82] as $percentage) { + runDiskUsageCheck($server, $percentage); + Carbon::setTestNow(now()->addMinutes(2)); + } + + Notification::assertSentToTimes($server->team, HighDiskUsage::class, 1); +});