From 99c840922c3d7161611493c967cc73263d33078e Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Tue, 29 Sep 2026 11:29:39 +0200 Subject: [PATCH] fix(notifications): throttle repeated notifications per resource High disk usage notifications were sent every ~10 minutes on Coolify Cloud because the cache rate limiter did not hold across Sentinel-triggered checks. - Add a notification_throttles table and NotificationThrottle helper that atomically records when a notification was sent for a resource. - Add the ThrottledNotification contract; Team::notify() skips a throttled notification that was already sent within its interval. - High disk usage: configurable per-server interval (server_disk_usage_notification_interval_hours, default 24h) in Server > Advanced and the servers API. - Container restarted (coolify-proxy, database TCP proxy): at most one notification per hour per server/database during a restart loop. - Move the unreachable and missing-backup markers into the new table and migrate existing data (servers.unreachable_notification_sent and scheduled_database_backups.missing_backup_notification_sent_at). - Regenerate OpenAPI files. --- .ai/lessons.md | 2 + .../Commands/CleanupUnreachableServers.php | 3 +- app/Contracts/ThrottledNotification.php | 19 +++ .../Controllers/Api/DatabasesController.php | 1 - .../Controllers/Api/ServersController.php | 7 +- app/Jobs/CheckMissingDatabaseBackupsJob.php | 11 +- app/Jobs/PushServerUpdateJob.php | 4 +- app/Jobs/ServerCheckJob.php | 2 +- app/Jobs/ServerStorageCheckJob.php | 19 +-- app/Livewire/Server/Advanced.php | 5 + app/Models/NotificationThrottle.php | 111 ++++++++++++++++ app/Models/ScheduledDatabaseBackup.php | 1 - app/Models/Server.php | 30 +++-- app/Models/ServerSetting.php | 2 + app/Models/Team.php | 21 ++- .../Container/ContainerRestarted.php | 19 ++- app/Notifications/Server/HighDiskUsage.php | 14 +- ...tion_interval_to_server_settings_table.php | 22 ++++ ...44_create_notification_throttles_table.php | 98 ++++++++++++++ openapi.json | 18 ++- openapi.yaml | 17 ++- .../views/livewire/server/advanced.blade.php | 6 +- tests/Feature/Api/LifecycleApisTest.php | 10 +- .../Feature/CleanupUnreachableServersTest.php | 11 +- .../CleanupUnsubscribedServersTest.php | 8 +- .../MissingDatabaseBackupNotificationTest.php | 5 +- tests/Feature/NotificationThrottleTest.php | 82 ++++++++++++ .../NotificationThrottlesMigrationTest.php | 68 ++++++++++ ...erStorageCheckNotificationIntervalTest.php | 122 ++++++++++++++++++ .../ServerReachabilityNotificationTest.php | 9 +- .../Feature/ServerUpdatePrivateKeyApiTest.php | 17 +++ .../SyncStripeSubscriptionsActionTest.php | 1 - tests/Feature/ThrottledNotificationTest.php | 102 +++++++++++++++ 33 files changed, 786 insertions(+), 81 deletions(-) create mode 100644 app/Contracts/ThrottledNotification.php create mode 100644 app/Models/NotificationThrottle.php create mode 100644 database/migrations/2026_09_29_082403_add_disk_usage_notification_interval_to_server_settings_table.php create mode 100644 database/migrations/2026_09_29_083244_create_notification_throttles_table.php create mode 100644 tests/Feature/NotificationThrottleTest.php create mode 100644 tests/Feature/NotificationThrottlesMigrationTest.php create mode 100644 tests/Feature/Server/ServerStorageCheckNotificationIntervalTest.php create mode 100644 tests/Feature/ThrottledNotificationTest.php diff --git a/.ai/lessons.md b/.ai/lessons.md index d68d184014..eff6d13e5c 100644 --- a/.ai/lessons.md +++ b/.ai/lessons.md @@ -5,6 +5,8 @@ - When a symptom matches an earlier fix, inspect that fix and prove why it no longer works before adding another workaround. - Test old reports against the current branch because later changes can make the report obsolete. - Use the same regression test before and after the production change so the result shows the behavior difference. +- Several dev instances can run from one checkout (see `./scripts/dev urls`). After you add a migration, run it on each running instance of that checkout, or those instances fail on the new code. +- The checkout is shared with other sessions. When the full suite fails, rerun each failing file alone and compare with a clean `git archive HEAD` copy before you connect a failure to your change. - Redirect browser test output to a file (`> /tmp/x.log 2>&1`); piping it (`| tail`) hangs because the Playwright server keeps the pipe open. - Call `visit()` directly in each `tests/v4/Browser` test body; Pest does not mark a test that only uses helper-wrapped `visit()` as a browser test, so it fails with `sendText() on null`. diff --git a/app/Console/Commands/CleanupUnreachableServers.php b/app/Console/Commands/CleanupUnreachableServers.php index 666e98a182..615dc21d20 100644 --- a/app/Console/Commands/CleanupUnreachableServers.php +++ b/app/Console/Commands/CleanupUnreachableServers.php @@ -3,6 +3,7 @@ namespace App\Console\Commands; use App\Models\Server; +use App\Notifications\Server\Unreachable; use Illuminate\Console\Command; class CleanupUnreachableServers extends Command @@ -14,7 +15,7 @@ class CleanupUnreachableServers extends Command public function handle() { echo "Running unreachable server cleanup...\n"; - $servers = Server::where('unreachable_count', '>=', 3)->where('unreachable_notification_sent', true)->where('updated_at', '<', now()->subDays(7))->get(); + $servers = Server::where('unreachable_count', '>=', 3)->whereHas('notificationThrottles', fn ($query) => $query->where('notification', Unreachable::class))->where('updated_at', '<', now()->subDays(7))->get(); if ($servers->count() > 0) { foreach ($servers as $server) { echo "Cleanup unreachable server ($server->id) with name $server->name"; diff --git a/app/Contracts/ThrottledNotification.php b/app/Contracts/ThrottledNotification.php new file mode 100644 index 0000000000..b578f75835 --- /dev/null +++ b/app/Contracts/ThrottledNotification.php @@ -0,0 +1,19 @@ +fill([ 'uuid' => new_public_id(), 'database_id' => $newDatabase->id, diff --git a/app/Http/Controllers/Api/ServersController.php b/app/Http/Controllers/Api/ServersController.php index 1a5d24b194..ef69f6b424 100644 --- a/app/Http/Controllers/Api/ServersController.php +++ b/app/Http/Controllers/Api/ServersController.php @@ -187,6 +187,7 @@ class ServersController extends Controller $server->load(['settings']); } + $server->append('unreachable_notification_sent'); $settings = $this->removeSensitiveDataFromSettings($server->settings); $server = $this->removeSensitiveData($server); data_set($server, 'settings', $settings); @@ -645,6 +646,7 @@ class ServersController extends Controller 'deployment_queue_limit' => ['type' => 'integer', 'description' => 'Maximum number of queued deployments.'], 'server_disk_usage_notification_threshold' => ['type' => 'integer', 'description' => 'Server disk usage notification threshold (%).'], 'server_disk_usage_check_frequency' => ['type' => 'string', 'description' => 'Cron expression for disk usage check frequency.'], + 'server_disk_usage_notification_interval_hours' => ['type' => 'integer', 'description' => 'Minimum hours between high disk usage notifications (1-720). Default: 24.'], 'connection_timeout' => ['type' => 'integer', 'description' => 'SSH connection timeout in seconds (1-300). Default: 10.'], ], ), @@ -682,7 +684,7 @@ class ServersController extends Controller )] public function update_server(Request $request) { - $allowedFields = ['name', 'description', 'ip', 'port', 'user', 'private_key_uuid', 'server_role', 'instant_validate', 'proxy_type', 'concurrent_builds', 'dynamic_timeout', 'deployment_queue_limit', 'server_disk_usage_notification_threshold', 'server_disk_usage_check_frequency', 'connection_timeout', 'is_terminal_enabled']; + $allowedFields = ['name', 'description', 'ip', 'port', 'user', 'private_key_uuid', 'server_role', 'instant_validate', 'proxy_type', 'concurrent_builds', 'dynamic_timeout', 'deployment_queue_limit', 'server_disk_usage_notification_threshold', 'server_disk_usage_check_frequency', 'server_disk_usage_notification_interval_hours', 'connection_timeout', 'is_terminal_enabled']; $teamId = getTeamIdFromToken(); if (is_null($teamId)) { @@ -708,6 +710,7 @@ class ServersController extends Controller 'deployment_queue_limit' => 'integer|min:1', 'server_disk_usage_notification_threshold' => 'integer|min:1|max:100', 'server_disk_usage_check_frequency' => 'string', + 'server_disk_usage_notification_interval_hours' => 'integer|min:1|max:720', 'connection_timeout' => 'integer|min:1|max:300', 'is_terminal_enabled' => 'boolean|nullable', ], [ @@ -790,7 +793,7 @@ class ServersController extends Controller ]); } - $advancedSettings = $request->only(['concurrent_builds', 'dynamic_timeout', 'deployment_queue_limit', 'server_disk_usage_notification_threshold', 'server_disk_usage_check_frequency', 'connection_timeout']); + $advancedSettings = $request->only(['concurrent_builds', 'dynamic_timeout', 'deployment_queue_limit', 'server_disk_usage_notification_threshold', 'server_disk_usage_check_frequency', 'server_disk_usage_notification_interval_hours', 'connection_timeout']); if (! empty($advancedSettings)) { $server->settings()->update(array_filter($advancedSettings, fn ($value) => ! is_null($value))); } diff --git a/app/Jobs/CheckMissingDatabaseBackupsJob.php b/app/Jobs/CheckMissingDatabaseBackupsJob.php index 06ba79a33f..957868f44d 100644 --- a/app/Jobs/CheckMissingDatabaseBackupsJob.php +++ b/app/Jobs/CheckMissingDatabaseBackupsJob.php @@ -2,6 +2,7 @@ namespace App\Jobs; +use App\Models\NotificationThrottle; use App\Models\ScheduledDatabaseBackup; use App\Notifications\Database\BackupMissing; use Illuminate\Bus\Queueable; @@ -39,10 +40,6 @@ class CheckMissingDatabaseBackupsJob implements ShouldBeEncrypted, ShouldBeUniqu return; } - if ($backup->missing_backup_notification_sent_at?->greaterThanOrEqualTo($lastActivityAt)) { - return; - } - if (! $backup->team) { Log::warning("Cannot send missing backup notification for backup {$backup->id}: team not found"); @@ -53,7 +50,11 @@ class CheckMissingDatabaseBackupsJob implements ShouldBeEncrypted, ShouldBeUniqu return; } + // Send once for each period without backup activity. + if (! NotificationThrottle::claim($backup, BackupMissing::class, $lastActivityAt)) { + return; + } + $backup->team->notify(new BackupMissing($backup, $lastExecutionAt)); - $backup->forceFill(['missing_backup_notification_sent_at' => now()])->save(); } } diff --git a/app/Jobs/PushServerUpdateJob.php b/app/Jobs/PushServerUpdateJob.php index 1324c212ed..3224d3a5e1 100644 --- a/app/Jobs/PushServerUpdateJob.php +++ b/app/Jobs/PushServerUpdateJob.php @@ -799,7 +799,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced try { if (CheckProxy::run($this->server)) { StartProxy::run($this->server, async: false); - $this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server)); + $this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server)); } } catch (\Throwable $e) { } @@ -841,7 +841,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced })->first(); if (! $tcpProxyContainerFound) { StartDatabaseProxy::dispatch($database); - $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$database->name}", $this->server)); + $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$database->name}", $this->server, restartedResource: $database)); } } elseif ($this->isRunning($containerStatus) && ! $tcpProxy) { // Clean up orphaned proxy containers when is_public=false diff --git a/app/Jobs/ServerCheckJob.php b/app/Jobs/ServerCheckJob.php index 10faa7e9b8..4b7203648c 100644 --- a/app/Jobs/ServerCheckJob.php +++ b/app/Jobs/ServerCheckJob.php @@ -85,7 +85,7 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue $shouldStart = CheckProxy::run($this->server); if ($shouldStart) { StartProxy::run($this->server, async: false); - $this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server)); + $this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server)); } } catch (\Throwable $e) { } diff --git a/app/Jobs/ServerStorageCheckJob.php b/app/Jobs/ServerStorageCheckJob.php index 51426d880a..a457207951 100644 --- a/app/Jobs/ServerStorageCheckJob.php +++ b/app/Jobs/ServerStorageCheckJob.php @@ -10,8 +10,8 @@ use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; +use Illuminate\Queue\TimeoutExceededException; use Illuminate\Support\Facades\Log; -use Illuminate\Support\Facades\RateLimiter; use Laravel\Horizon\Contracts\Silenced; class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue, Silenced @@ -31,7 +31,7 @@ class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue, Silenced public function failed(?\Throwable $exception): void { - if ($exception instanceof \Illuminate\Queue\TimeoutExceededException) { + if ($exception instanceof TimeoutExceededException) { Log::warning('ServerStorageCheckJob timed out', [ 'server_id' => $this->server->id, 'server_name' => $this->server->name, @@ -58,20 +58,7 @@ class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue, Silenced return 'No percentage could be retrieved.'; } if ($this->percentage > $serverDiskUsageNotificationThreshold) { - $executed = RateLimiter::attempt( - 'high-disk-usage:'.$this->server->id, - $maxAttempts = 0, - function () use ($team, $serverDiskUsageNotificationThreshold) { - $team->notify(new HighDiskUsage($this->server, $this->percentage, $serverDiskUsageNotificationThreshold)); - }, - $decaySeconds = 3600, - ); - - if (! $executed) { - return 'Too many messages sent!'; - } - } else { - RateLimiter::hit('high-disk-usage:'.$this->server->id, 600); + $team->notify(new HighDiskUsage($this->server, $this->percentage, $serverDiskUsageNotificationThreshold)); } } catch (\Throwable $e) { return handleError($e); diff --git a/app/Livewire/Server/Advanced.php b/app/Livewire/Server/Advanced.php index 895ce34e79..34847a12e6 100644 --- a/app/Livewire/Server/Advanced.php +++ b/app/Livewire/Server/Advanced.php @@ -18,6 +18,9 @@ class Advanced extends Component #[Validate(['required', 'integer', 'min:1', 'max:99'])] public int|string $serverDiskUsageNotificationThreshold = 50; + #[Validate(['required', 'integer', 'min:1', 'max:720'])] + public int|string $serverDiskUsageNotificationIntervalHours = 24; + #[Validate(['required', 'integer', 'min:1'])] public int|string $concurrentBuilds = 1; @@ -52,6 +55,7 @@ class Advanced extends Component $this->server->settings->backup_compression_cpu_percentage = $this->backupCompressionCpuPercentage; $this->server->settings->server_disk_usage_notification_threshold = $this->serverDiskUsageNotificationThreshold; $this->server->settings->server_disk_usage_check_frequency = $this->serverDiskUsageCheckFrequency; + $this->server->settings->server_disk_usage_notification_interval_hours = $this->serverDiskUsageNotificationIntervalHours; $this->server->settings->save(); } else { $this->concurrentBuilds = $this->server->settings->concurrent_builds; @@ -60,6 +64,7 @@ class Advanced extends Component $this->backupCompressionCpuPercentage = $this->server->settings->backup_compression_cpu_percentage; $this->serverDiskUsageNotificationThreshold = $this->server->settings->server_disk_usage_notification_threshold; $this->serverDiskUsageCheckFrequency = $this->server->settings->server_disk_usage_check_frequency; + $this->serverDiskUsageNotificationIntervalHours = $this->server->settings->server_disk_usage_notification_interval_hours; } } diff --git a/app/Models/NotificationThrottle.php b/app/Models/NotificationThrottle.php new file mode 100644 index 0000000000..46340f9fd3 --- /dev/null +++ b/app/Models/NotificationThrottle.php @@ -0,0 +1,111 @@ + 'datetime', + ]; + } + + public function notifiable(): MorphTo + { + return $this->morphTo(); + } + + /** + * Atomically reserve the right to send a notification. + * + * Returns true when the notification was never sent, or when it was last sent + * before $sentBefore. Without $sentBefore, it is sent only once until released. + */ + public static function claim(Model $notifiable, string $notification, ?CarbonInterface $sentBefore = null): bool + { + if ($sentBefore !== null) { + $renewed = static::query() + ->whereMorphedTo('notifiable', $notifiable) + ->where('notification', $notification) + ->where('sent_at', '<', $sentBefore) + ->update(['sent_at' => now()]); + + if ($renewed === 1) { + return true; + } + } + + return static::query()->insertOrIgnore([ + 'notifiable_type' => $notifiable->getMorphClass(), + 'notifiable_id' => $notifiable->getKey(), + 'notification' => $notification, + 'sent_at' => now(), + ]) === 1; + } + + /** + * Claim the throttle a notification declares. Notifications without a throttle subject are always allowed. + */ + public static function claimFor(ThrottledNotification $notification): bool + { + $subject = $notification->throttleSubject(); + if ($subject === null) { + return true; + } + + return static::claim($subject, $notification::class, now()->subMinutes($notification->throttleIntervalMinutes())); + } + + /** + * Mark a notification as sent without sending it. + */ + public static function record(Model $notifiable, string $notification): void + { + static::query()->updateOrInsert( + [ + 'notifiable_type' => $notifiable->getMorphClass(), + 'notifiable_id' => $notifiable->getKey(), + 'notification' => $notification, + ], + ['sent_at' => now()], + ); + } + + /** + * Forget a sent notification. Returns true when a record was removed. + */ + public static function release(Model $notifiable, string $notification): bool + { + return static::query() + ->whereMorphedTo('notifiable', $notifiable) + ->where('notification', $notification) + ->delete() > 0; + } + + public static function wasSent(Model $notifiable, string $notification): bool + { + return static::query() + ->whereMorphedTo('notifiable', $notifiable) + ->where('notification', $notification) + ->exists(); + } +} diff --git a/app/Models/ScheduledDatabaseBackup.php b/app/Models/ScheduledDatabaseBackup.php index 73c162ea81..b44d1b16f4 100644 --- a/app/Models/ScheduledDatabaseBackup.php +++ b/app/Models/ScheduledDatabaseBackup.php @@ -18,7 +18,6 @@ class ScheduledDatabaseBackup extends BaseModel 'database_backup_retention_max_storage_locally' => 'float', 'database_backup_retention_max_storage_s3' => 'float', 'missing_backup_notification_days' => 'integer', - 'missing_backup_notification_sent_at' => 'datetime', 'last_execution_at' => 'datetime', 'next_run_at' => 'datetime', ]; diff --git a/app/Models/Server.php b/app/Models/Server.php index 00eaad8839..9f4e6667d8 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -29,6 +29,7 @@ use App\Traits\HasSafeStringAttribute; use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Factories\HasFactory; +use Illuminate\Database\Eloquent\Relations\MorphMany; use Illuminate\Database\Eloquent\SoftDeletes; use Illuminate\Support\Carbon; use Illuminate\Support\Collection; @@ -235,6 +236,7 @@ class Server extends BaseModel }); $server->settings()->delete(); $server->sslCertificates()->delete(); + $server->notificationThrottles()->delete(); }); static::updated(function () { @@ -280,7 +282,6 @@ class Server extends BaseModel 'logdrain_newrelic_license_key' => 'encrypted', 'delete_unused_volumes' => 'boolean', 'delete_unused_networks' => 'boolean', - 'unreachable_notification_sent' => 'boolean', 'force_disabled' => 'boolean', 'sentinel_waiting_since' => 'datetime', ]; @@ -1526,6 +1527,19 @@ $siteAddress { return $this->belongsTo(CloudProviderToken::class); } + public function notificationThrottles(): MorphMany + { + return $this->morphMany(NotificationThrottle::class, 'notifiable'); + } + + /** + * True while an Unreachable notification was sent and no Reachable notification followed. + */ + protected function unreachableNotificationSent(): Attribute + { + return Attribute::get(fn (): bool => NotificationThrottle::wasSent($this, Unreachable::class)); + } + public function sslCertificates() { return $this->hasMany(SslCertificate::class); @@ -1777,18 +1791,16 @@ $siteAddress { public function sendReachableNotification() { - $this->unreachable_notification_sent = false; - $this->save(); - $this->refresh(); - $this->team->notify(new Reachable($this)); + if (NotificationThrottle::release($this, Unreachable::class)) { + $this->team->notify(new Reachable($this)); + } } public function sendUnreachableNotification() { - $this->unreachable_notification_sent = true; - $this->save(); - $this->refresh(); - $this->team->notify(new Unreachable($this)); + if (NotificationThrottle::claim($this, Unreachable::class)) { + $this->team->notify(new Unreachable($this)); + } } public function validateConnection(bool $justCheckingNewKey = false) diff --git a/app/Models/ServerSetting.php b/app/Models/ServerSetting.php index 53674dad92..defb842dda 100644 --- a/app/Models/ServerSetting.php +++ b/app/Models/ServerSetting.php @@ -117,6 +117,7 @@ class ServerSetting extends Model 'server_disk_usage_notification_threshold', 'is_sentinel_debug_enabled', 'server_disk_usage_check_frequency', + 'server_disk_usage_notification_interval_hours', 'is_terminal_enabled', 'deployment_queue_limit', 'backup_compression_cpu_percentage', @@ -170,6 +171,7 @@ class ServerSetting extends Model 'compose_version_checked_at' => 'datetime', 'backup_compression_cpu_percentage' => 'integer', 'docker_cleanup_next_run_at' => 'datetime', + 'server_disk_usage_notification_interval_hours' => 'integer', ]; /** diff --git a/app/Models/Team.php b/app/Models/Team.php index 6ce1c1d3c1..da08882f6b 100644 --- a/app/Models/Team.php +++ b/app/Models/Team.php @@ -3,11 +3,13 @@ namespace App\Models; use App\Actions\User\RevokeUserTeamTokens; +use App\Contracts\ThrottledNotification; use App\Events\ServerReachabilityChanged; use App\Notifications\Channels\SendsDiscord; use App\Notifications\Channels\SendsEmail; use App\Notifications\Channels\SendsPushover; use App\Notifications\Channels\SendsSlack; +use App\Notifications\Server\Unreachable; use App\Traits\Auditable; use App\Traits\HasNotificationSettings; use App\Traits\HasSafeStringAttribute; @@ -43,7 +45,10 @@ use OpenApi\Attributes as OA; class Team extends Model implements SendsDiscord, SendsEmail, SendsPushover, SendsSlack { - use Auditable, HasFactory, HasNotificationSettings, HasSafeStringAttribute, Notifiable; + use Auditable, HasFactory, HasNotificationSettings, HasSafeStringAttribute; + use Notifiable { + notify as sendNotification; + } protected $fillable = [ 'name', @@ -273,11 +278,23 @@ class Team extends Model implements SendsDiscord, SendsEmail, SendsPushover, Sen ]); ServerReachabilityChanged::dispatch($server); $server->unreachable_count = 3; - $server->unreachable_notification_sent = true; $server->save(); + NotificationThrottle::record($server, Unreachable::class); } } + /** + * Send a notification, unless it is throttled and was already sent within its interval. + */ + public function notify($instance): void + { + if ($instance instanceof ThrottledNotification && ! NotificationThrottle::claimFor($instance)) { + return; + } + + $this->sendNotification($instance); + } + public function environment_variables() { return $this->hasMany(SharedEnvironmentVariable::class)->where('type', 'team'); diff --git a/app/Notifications/Container/ContainerRestarted.php b/app/Notifications/Container/ContainerRestarted.php index 2d7eb58b50..012f997754 100644 --- a/app/Notifications/Container/ContainerRestarted.php +++ b/app/Notifications/Container/ContainerRestarted.php @@ -2,16 +2,21 @@ namespace App\Notifications\Container; +use App\Contracts\ThrottledNotification; use App\Models\Server; use App\Notifications\CustomEmailNotification; use App\Notifications\Dto\DiscordMessage; use App\Notifications\Dto\PushoverMessage; use App\Notifications\Dto\SlackMessage; +use Illuminate\Database\Eloquent\Model; use Illuminate\Notifications\Messages\MailMessage; -class ContainerRestarted extends CustomEmailNotification +class ContainerRestarted extends CustomEmailNotification implements ThrottledNotification { - public function __construct(public string $name, public Server $server, public ?string $url = null) + /** + * When $restartedResource is set, a restart loop of that resource sends at most one notification per hour. + */ + public function __construct(public string $name, public Server $server, public ?string $url = null, public ?Model $restartedResource = null) { $this->onQueue('high'); } @@ -21,6 +26,16 @@ class ContainerRestarted extends CustomEmailNotification return $notifiable->getEnabledChannels('status_change'); } + public function throttleSubject(): ?Model + { + return $this->restartedResource; + } + + public function throttleIntervalMinutes(): int + { + return 60; + } + public function toMail(): MailMessage { $mail = new MailMessage; diff --git a/app/Notifications/Server/HighDiskUsage.php b/app/Notifications/Server/HighDiskUsage.php index 149d1bbc86..be0517b759 100644 --- a/app/Notifications/Server/HighDiskUsage.php +++ b/app/Notifications/Server/HighDiskUsage.php @@ -2,14 +2,16 @@ namespace App\Notifications\Server; +use App\Contracts\ThrottledNotification; use App\Models\Server; use App\Notifications\CustomEmailNotification; use App\Notifications\Dto\DiscordMessage; use App\Notifications\Dto\PushoverMessage; use App\Notifications\Dto\SlackMessage; +use Illuminate\Database\Eloquent\Model; use Illuminate\Notifications\Messages\MailMessage; -class HighDiskUsage extends CustomEmailNotification +class HighDiskUsage extends CustomEmailNotification implements ThrottledNotification { public function __construct(public Server $server, public int $disk_usage, public int $server_disk_usage_notification_threshold) { @@ -21,6 +23,16 @@ class HighDiskUsage extends CustomEmailNotification return $notifiable->getEnabledChannels('server_disk_usage'); } + public function throttleSubject(): ?Model + { + return $this->server; + } + + public function throttleIntervalMinutes(): int + { + return max(1, (int) data_get($this->server, 'settings.server_disk_usage_notification_interval_hours', 24)) * 60; + } + public function toMail(): MailMessage { $mail = new MailMessage; diff --git a/database/migrations/2026_09_29_082403_add_disk_usage_notification_interval_to_server_settings_table.php b/database/migrations/2026_09_29_082403_add_disk_usage_notification_interval_to_server_settings_table.php new file mode 100644 index 0000000000..6ec4a5a312 --- /dev/null +++ b/database/migrations/2026_09_29_082403_add_disk_usage_notification_interval_to_server_settings_table.php @@ -0,0 +1,22 @@ +integer('server_disk_usage_notification_interval_hours')->default(24); + }); + } + + public function down(): void + { + Schema::table('server_settings', function (Blueprint $table) { + $table->dropColumn('server_disk_usage_notification_interval_hours'); + }); + } +}; diff --git a/database/migrations/2026_09_29_083244_create_notification_throttles_table.php b/database/migrations/2026_09_29_083244_create_notification_throttles_table.php new file mode 100644 index 0000000000..e11a14a567 --- /dev/null +++ b/database/migrations/2026_09_29_083244_create_notification_throttles_table.php @@ -0,0 +1,98 @@ +id(); + $table->morphs('notifiable'); + $table->string('notification'); + $table->timestampTz('sent_at'); + $table->unique(['notifiable_type', 'notifiable_id', 'notification']); + }); + + DB::table('servers') + ->where('unreachable_notification_sent', true) + ->orderBy('id') + ->select(['id', 'updated_at']) + ->chunk(500, function ($servers) { + DB::table('notification_throttles')->insertOrIgnore($servers->map(fn ($server) => [ + 'notifiable_type' => self::SERVER, + 'notifiable_id' => $server->id, + 'notification' => self::UNREACHABLE, + 'sent_at' => $server->updated_at ?? now(), + ])->all()); + }); + + DB::table('scheduled_database_backups') + ->whereNotNull('missing_backup_notification_sent_at') + ->orderBy('id') + ->select(['id', 'missing_backup_notification_sent_at']) + ->chunk(500, function ($backups) { + DB::table('notification_throttles')->insertOrIgnore($backups->map(fn ($backup) => [ + 'notifiable_type' => self::BACKUP, + 'notifiable_id' => $backup->id, + 'notification' => self::BACKUP_MISSING, + 'sent_at' => $backup->missing_backup_notification_sent_at, + ])->all()); + }); + + Schema::table('servers', function (Blueprint $table) { + $table->dropColumn('unreachable_notification_sent'); + }); + Schema::table('scheduled_database_backups', function (Blueprint $table) { + $table->dropColumn('missing_backup_notification_sent_at'); + }); + } + + public function down(): void + { + Schema::table('servers', function (Blueprint $table) { + $table->boolean('unreachable_notification_sent')->default(false); + }); + Schema::table('scheduled_database_backups', function (Blueprint $table) { + $table->timestamp('missing_backup_notification_sent_at')->nullable(); + }); + + DB::table('notification_throttles') + ->where('notifiable_type', self::SERVER) + ->where('notification', self::UNREACHABLE) + ->orderBy('id') + ->chunk(500, function ($throttles) { + DB::table('servers') + ->whereIn('id', $throttles->pluck('notifiable_id')) + ->update(['unreachable_notification_sent' => true]); + }); + + DB::table('notification_throttles') + ->where('notifiable_type', self::BACKUP) + ->where('notification', self::BACKUP_MISSING) + ->orderBy('id') + ->chunk(500, function ($throttles) { + foreach ($throttles as $throttle) { + DB::table('scheduled_database_backups') + ->where('id', $throttle->notifiable_id) + ->update(['missing_backup_notification_sent_at' => $throttle->sent_at]); + } + }); + + Schema::dropIfExists('notification_throttles'); + } +}; diff --git a/openapi.json b/openapi.json index 512f94b9e6..8087d32199 100644 --- a/openapi.json +++ b/openapi.json @@ -3531,7 +3531,7 @@ "Applications" ], "summary": "Get application logs.", - "description": "Get application logs by UUID.", + "description": "Get application logs by UUID. Requires the `read:sensitive` or `root` token ability.", "operationId": "get-application-logs-by-uuid", "parameters": [ { @@ -3616,7 +3616,7 @@ "Applications" ], "summary": "Get preview application logs.", - "description": "Get runtime container logs for a preview deployment by application UUID and pull request ID.", + "description": "Get runtime container logs for a preview deployment by application UUID and pull request ID. Requires the `read:sensitive` or `root` token ability.", "operationId": "get-preview-application-logs-by-pull-request-id", "parameters": [ { @@ -8293,7 +8293,7 @@ "Databases" ], "summary": "Get database logs.", - "description": "Get database logs by UUID.", + "description": "Get database logs by UUID. Requires the `read:sensitive` or `root` token ability.", "operationId": "get-database-logs-by-uuid", "parameters": [ { @@ -16766,6 +16766,10 @@ "type": "string", "description": "Cron expression for disk usage check frequency." }, + "server_disk_usage_notification_interval_hours": { + "type": "integer", + "description": "Minimum hours between high disk usage notifications (1-720). Default: 24." + }, "connection_timeout": { "type": "integer", "description": "SSH connection timeout in seconds (1-300). Default: 10." @@ -17265,7 +17269,7 @@ "Service applications" ], "summary": "Get service application logs", - "description": "Get Docker logs for a single compose service container.", + "description": "Get Docker logs for a single compose service container. Requires the `read:sensitive` or `root` token ability.", "operationId": "get-service-application-logs-by-service-and-app-uuid", "parameters": [ { @@ -17350,7 +17354,7 @@ "Service applications" ], "summary": "Get service application logs", - "description": "Get Docker logs for a single compose service container.", + "description": "Get Docker logs for a single compose service container. Requires the `read:sensitive` or `root` token ability.", "operationId": "post-service-application-logs-by-service-and-app-uuid", "parameters": [ { @@ -18002,7 +18006,7 @@ "Service databases" ], "summary": "Get service database logs", - "description": "Get Docker logs for a compose database container.", + "description": "Get Docker logs for a compose database container. Requires the `read:sensitive` or `root` token ability.", "operationId": "get-service-database-logs-by-service-and-database-uuid", "parameters": [ { @@ -18847,7 +18851,7 @@ "Services" ], "summary": "Get service logs.", - "description": "Get logs for a specific service sub-resource by service UUID. The `sub_service_name` query parameter must match the `name` field of one of the service applications or databases returned by `GET \/services\/{uuid}`.", + "description": "Get logs for a specific service sub-resource by service UUID. The `sub_service_name` query parameter must match the `name` field of one of the service applications or databases returned by `GET \/services\/{uuid}`. Requires the `read:sensitive` or `root` token ability.", "operationId": "get-service-logs-by-uuid", "parameters": [ { diff --git a/openapi.yaml b/openapi.yaml index e6261818b1..a78f236afc 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -2325,7 +2325,7 @@ paths: tags: - Applications summary: 'Get application logs.' - description: 'Get application logs by UUID.' + description: 'Get application logs by UUID. Requires the `read:sensitive` or `root` token ability.' operationId: get-application-logs-by-uuid parameters: - @@ -2383,7 +2383,7 @@ paths: tags: - Applications summary: 'Get preview application logs.' - description: 'Get runtime container logs for a preview deployment by application UUID and pull request ID.' + description: 'Get runtime container logs for a preview deployment by application UUID and pull request ID. Requires the `read:sensitive` or `root` token ability.' operationId: get-preview-application-logs-by-pull-request-id parameters: - @@ -5469,7 +5469,7 @@ paths: tags: - Databases summary: 'Get database logs.' - description: 'Get database logs by UUID.' + description: 'Get database logs by UUID. Requires the `read:sensitive` or `root` token ability.' operationId: get-database-logs-by-uuid parameters: - @@ -10647,6 +10647,9 @@ paths: server_disk_usage_check_frequency: type: string description: 'Cron expression for disk usage check frequency.' + server_disk_usage_notification_interval_hours: + type: integer + description: 'Minimum hours between high disk usage notifications (1-720). Default: 24.' connection_timeout: type: integer description: 'SSH connection timeout in seconds (1-300). Default: 10.' @@ -10931,7 +10934,7 @@ paths: tags: - 'Service applications' summary: 'Get service application logs' - description: 'Get Docker logs for a single compose service container.' + description: 'Get Docker logs for a single compose service container. Requires the `read:sensitive` or `root` token ability.' operationId: get-service-application-logs-by-service-and-app-uuid parameters: - @@ -10989,7 +10992,7 @@ paths: tags: - 'Service applications' summary: 'Get service application logs' - description: 'Get Docker logs for a single compose service container.' + description: 'Get Docker logs for a single compose service container. Requires the `read:sensitive` or `root` token ability.' operationId: post-service-application-logs-by-service-and-app-uuid parameters: - @@ -11418,7 +11421,7 @@ paths: tags: - 'Service databases' summary: 'Get service database logs' - description: 'Get Docker logs for a compose database container.' + description: 'Get Docker logs for a compose database container. Requires the `read:sensitive` or `root` token ability.' operationId: get-service-database-logs-by-service-and-database-uuid parameters: - @@ -11912,7 +11915,7 @@ paths: tags: - Services summary: 'Get service logs.' - description: 'Get logs for a specific service sub-resource by service UUID. The `sub_service_name` query parameter must match the `name` field of one of the service applications or databases returned by `GET /services/{uuid}`.' + description: 'Get logs for a specific service sub-resource by service UUID. The `sub_service_name` query parameter must match the `name` field of one of the service applications or databases returned by `GET /services/{uuid}`. Requires the `read:sensitive` or `root` token ability.' operationId: get-service-logs-by-uuid parameters: - diff --git a/resources/views/livewire/server/advanced.blade.php b/resources/views/livewire/server/advanced.blade.php index 48f63f02e9..a7244869f3 100644 --- a/resources/views/livewire/server/advanced.blade.php +++ b/resources/views/livewire/server/advanced.blade.php @@ -14,7 +14,7 @@ -
+
@@ -22,6 +22,10 @@ id="serverDiskUsageNotificationThreshold" type="number" min="1" max="99" label="Notification threshold" required helper="Notify the team when root filesystem usage exceeds this percentage." /> +
diff --git a/tests/Feature/Api/LifecycleApisTest.php b/tests/Feature/Api/LifecycleApisTest.php index 95004550b8..d89dcf414d 100644 --- a/tests/Feature/Api/LifecycleApisTest.php +++ b/tests/Feature/Api/LifecycleApisTest.php @@ -7,6 +7,7 @@ use App\Models\CloudInitScript; use App\Models\Environment; use App\Models\InstanceSettings; use App\Models\LocalPersistentVolume; +use App\Models\NotificationThrottle; use App\Models\Project; use App\Models\ScheduledTask; use App\Models\Server; @@ -15,6 +16,7 @@ use App\Models\StandaloneDocker; use App\Models\StandalonePostgresql; use App\Models\Team; use App\Models\User; +use App\Notifications\Database\BackupMissing; use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Support\Facades\Bus; use Illuminate\Support\Facades\Queue; @@ -144,10 +146,8 @@ describe('POST /api/v1/databases/{uuid}/clone', function () { 'frequency' => '0 0 * * *', 'save_s3' => false, ]); - $backup->forceFill([ - 'last_execution_at' => now()->subDay(), - 'missing_backup_notification_sent_at' => now(), - ])->save(); + $backup->forceFill(['last_execution_at' => now()->subDay()])->save(); + NotificationThrottle::record($backup, BackupMissing::class); $response = $this->withHeaders($this->headers) ->postJson("/api/v1/databases/{$database->uuid}/clone", [ @@ -166,7 +166,7 @@ describe('POST /api/v1/databases/{uuid}/clone', function () { ->and($cloned->destination_id)->toBe($this->destination->id) ->and(str($cloned->status)->startsWith('exited'))->toBeTrue() ->and($clonedBackup->last_execution_at)->toBeNull() - ->and($clonedBackup->missing_backup_notification_sent_at)->toBeNull(); + ->and(NotificationThrottle::wasSent($clonedBackup, BackupMissing::class))->toBeFalse(); }); test('creates renamed volumes when cloning a database with clone_volumes', function () { diff --git a/tests/Feature/CleanupUnreachableServersTest.php b/tests/Feature/CleanupUnreachableServersTest.php index 8849b1ca01..2c58946891 100644 --- a/tests/Feature/CleanupUnreachableServersTest.php +++ b/tests/Feature/CleanupUnreachableServersTest.php @@ -1,7 +1,9 @@ = 3 $server = Server::factory()->create([ 'team_id' => $team->id, 'unreachable_count' => 50, - 'unreachable_notification_sent' => true, 'updated_at' => now()->subDays(8), ]); + NotificationThrottle::record($server, Unreachable::class); $originalIp = (string) $server->ip; @@ -34,9 +36,9 @@ it('overwrites the IP with 1.2.3.4 on cloud for servers with unreachable_count > $server = Server::factory()->create([ 'team_id' => $team->id, 'unreachable_count' => 50, - 'unreachable_notification_sent' => true, 'updated_at' => now()->subDays(8), ]); + NotificationThrottle::record($server, Unreachable::class); $this->artisan('cleanup:unreachable-servers')->assertSuccessful(); @@ -49,9 +51,9 @@ it('does not clean up servers with unreachable_count less than 3', function () { $server = Server::factory()->create([ 'team_id' => $team->id, 'unreachable_count' => 2, - 'unreachable_notification_sent' => true, 'updated_at' => now()->subDays(8), ]); + NotificationThrottle::record($server, Unreachable::class); $originalIp = (string) $server->ip; @@ -67,9 +69,9 @@ it('does not clean up servers updated within 7 days', function () { $server = Server::factory()->create([ 'team_id' => $team->id, 'unreachable_count' => 10, - 'unreachable_notification_sent' => true, 'updated_at' => now()->subDays(3), ]); + NotificationThrottle::record($server, Unreachable::class); $originalIp = (string) $server->ip; @@ -85,7 +87,6 @@ it('does not clean up servers without notification sent', function () { $server = Server::factory()->create([ 'team_id' => $team->id, 'unreachable_count' => 10, - 'unreachable_notification_sent' => false, 'updated_at' => now()->subDays(8), ]); diff --git a/tests/Feature/CleanupUnsubscribedServersTest.php b/tests/Feature/CleanupUnsubscribedServersTest.php index 53b0160d8a..edf6c63a0b 100644 --- a/tests/Feature/CleanupUnsubscribedServersTest.php +++ b/tests/Feature/CleanupUnsubscribedServersTest.php @@ -1,8 +1,10 @@ create([ 'team_id' => $team->id, 'unreachable_count' => 0, - 'unreachable_notification_sent' => false, ]); $team->subscriptionEnded(); @@ -34,9 +35,9 @@ it('cleans up unsubscribed server IP after 7 days via cleanup command', function $server = Server::factory()->create([ 'team_id' => $team->id, 'unreachable_count' => 3, - 'unreachable_notification_sent' => true, 'updated_at' => now()->subDays(8), ]); + NotificationThrottle::record($server, Unreachable::class); $this->artisan('cleanup:unreachable-servers')->assertSuccessful(); @@ -49,9 +50,9 @@ it('does not clean up unsubscribed server IP within 7 day grace period', functio $server = Server::factory()->create([ 'team_id' => $team->id, 'unreachable_count' => 3, - 'unreachable_notification_sent' => true, 'updated_at' => now()->subDays(3), ]); + NotificationThrottle::record($server, Unreachable::class); $originalIp = (string) $server->ip; @@ -70,7 +71,6 @@ it('does not affect servers with active subscriptions', function () { $server = Server::factory()->create([ 'team_id' => $team->id, 'unreachable_count' => 0, - 'unreachable_notification_sent' => false, ]); $originalCount = $server->unreachable_count; diff --git a/tests/Feature/MissingDatabaseBackupNotificationTest.php b/tests/Feature/MissingDatabaseBackupNotificationTest.php index f4f2efd191..9ec50feae3 100644 --- a/tests/Feature/MissingDatabaseBackupNotificationTest.php +++ b/tests/Feature/MissingDatabaseBackupNotificationTest.php @@ -1,6 +1,7 @@ handle(); Notification::assertSentTo($team, BackupMissing::class, fn (BackupMissing $notification) => $notification->backup->is($backup)); - expect($backup->fresh()->missing_backup_notification_sent_at)->not->toBeNull(); + expect(NotificationThrottle::wasSent($backup, BackupMissing::class))->toBeTrue(); }); it('does not notify for recent disabled or unconfigured backup schedules', function () { @@ -114,7 +115,7 @@ it('waits to mark an incident sent until a notification channel is enabled', fun $backup = missingBackupSchedule($team); (new CheckMissingDatabaseBackupsJob)->handle(); - expect($backup->fresh()->missing_backup_notification_sent_at)->toBeNull(); + expect(NotificationThrottle::wasSent($backup, BackupMissing::class))->toBeFalse(); $team->emailNotificationSettings->update([ 'smtp_enabled' => true, diff --git a/tests/Feature/NotificationThrottleTest.php b/tests/Feature/NotificationThrottleTest.php new file mode 100644 index 0000000000..b6d3d881f0 --- /dev/null +++ b/tests/Feature/NotificationThrottleTest.php @@ -0,0 +1,82 @@ +team = Team::factory()->create(); + $this->server = Server::factory()->create(['team_id' => $this->team->id]); +}); + +afterEach(function () { + Carbon::setTestNow(); +}); + +it('claims a notification only once until it is released', function () { + expect(NotificationThrottle::claim($this->server, Unreachable::class))->toBeTrue() + ->and(NotificationThrottle::claim($this->server, Unreachable::class))->toBeFalse() + ->and($this->server->unreachable_notification_sent)->toBeTrue() + ->and(NotificationThrottle::release($this->server, Unreachable::class))->toBeTrue() + ->and(NotificationThrottle::release($this->server, Unreachable::class))->toBeFalse() + ->and($this->server->unreachable_notification_sent)->toBeFalse() + ->and(NotificationThrottle::claim($this->server, Unreachable::class))->toBeTrue(); +}); + +it('claims a notification again only when it was sent before the given time', function () { + Carbon::setTestNow(Carbon::parse('2026-09-29 12:00:00', 'UTC')); + + expect(NotificationThrottle::claim($this->server, HighDiskUsage::class, now()->subHours(6)))->toBeTrue(); + + Carbon::setTestNow(now()->addHours(5)); + expect(NotificationThrottle::claim($this->server, HighDiskUsage::class, now()->subHours(6)))->toBeFalse(); + + Carbon::setTestNow(now()->addHours(2)); + expect(NotificationThrottle::claim($this->server, HighDiskUsage::class, now()->subHours(6)))->toBeTrue() + ->and(NotificationThrottle::query()->count())->toBe(1); +}); + +it('keeps throttles separate for each notification and resource', function () { + $otherServer = Server::factory()->create(['team_id' => $this->team->id]); + + NotificationThrottle::record($this->server, Unreachable::class); + + expect(NotificationThrottle::claim($this->server, HighDiskUsage::class))->toBeTrue() + ->and(NotificationThrottle::claim($otherServer, Unreachable::class))->toBeTrue() + ->and(NotificationThrottle::claim($this->server, Unreachable::class))->toBeFalse(); +}); + +it('removes the throttles of a server when the server is force deleted', function () { + NotificationThrottle::record($this->server, Unreachable::class); + + $this->server->forceDelete(); + + expect(NotificationThrottle::query()->count())->toBe(0); +}); + +it('keeps unreachable_notification_sent in the server API response', function () { + config()->set('app.maintenance.driver', 'file'); + config()->set('cache.default', 'array'); + InstanceSettings::forceCreate(['id' => 0, 'is_api_enabled' => true]); + $user = User::factory()->create(); + $this->team->members()->attach($user->id, ['role' => 'owner']); + session(['currentTeam' => $this->team]); + $token = $user->createToken('read-token', ['read']); + $token->accessToken->forceFill(['team_id' => $this->team->id])->save(); + + NotificationThrottle::record($this->server, Unreachable::class); + + $this->withHeaders(['Authorization' => 'Bearer '.$token->plainTextToken]) + ->getJson('/api/v1/servers/'.$this->server->uuid) + ->assertOk() + ->assertJsonPath('unreachable_notification_sent', true); +}); diff --git a/tests/Feature/NotificationThrottlesMigrationTest.php b/tests/Feature/NotificationThrottlesMigrationTest.php new file mode 100644 index 0000000000..04a641205d --- /dev/null +++ b/tests/Feature/NotificationThrottlesMigrationTest.php @@ -0,0 +1,68 @@ +create(); + $localhost = Server::factory()->create(['id' => 0, 'team_id' => $team->id]); + $notifiedServer = Server::factory()->create(['team_id' => $team->id]); + $quietServer = Server::factory()->create(['team_id' => $team->id]); + $notifiedBackup = ScheduledDatabaseBackup::create([ + 'enabled' => true, + 'frequency' => '0 0 * * *', + 'save_s3' => false, + 'database_type' => 'App\\Models\\StandalonePostgresql', + 'database_id' => 999, + 'team_id' => $team->id, + ]); + $quietBackup = $notifiedBackup->replicate(['uuid']); + $quietBackup->save(); + + $migration = require database_path('migrations/2026_09_29_083244_create_notification_throttles_table.php'); + $migration->down(); + + DB::table('servers')->whereIn('id', [$localhost->id, $notifiedServer->id])->update(['unreachable_notification_sent' => true]); + DB::table('scheduled_database_backups')->where('id', $notifiedBackup->id)->update(['missing_backup_notification_sent_at' => '2026-09-01 10:00:00']); + + $migration->up(); + + expect(Schema::hasColumn('servers', 'unreachable_notification_sent'))->toBeFalse() + ->and(Schema::hasColumn('scheduled_database_backups', 'missing_backup_notification_sent_at'))->toBeFalse() + ->and(NotificationThrottle::query()->count())->toBe(3) + ->and(NotificationThrottle::wasSent($localhost, Unreachable::class))->toBeTrue() + ->and(NotificationThrottle::wasSent($notifiedServer, Unreachable::class))->toBeTrue() + ->and(NotificationThrottle::wasSent($quietServer, Unreachable::class))->toBeFalse() + ->and(NotificationThrottle::wasSent($quietBackup, BackupMissing::class))->toBeFalse() + ->and(NotificationThrottle::query() + ->whereMorphedTo('notifiable', $notifiedBackup) + ->where('notification', BackupMissing::class) + ->sole() + ->sent_at + ->toDateTimeString())->toBe('2026-09-01 10:00:00'); + + $migration->down(); + + expect(Schema::hasTable('notification_throttles'))->toBeFalse() + ->and(DB::table('servers')->where('unreachable_notification_sent', true)->pluck('id')->sort()->values()->all()) + ->toBe([$localhost->id, $notifiedServer->id]) + ->and(DB::table('scheduled_database_backups')->where('id', $notifiedBackup->id)->value('missing_backup_notification_sent_at')) + ->toStartWith('2026-09-01 10:00:00') + ->and(DB::table('scheduled_database_backups')->where('id', $quietBackup->id)->value('missing_backup_notification_sent_at')) + ->toBeNull(); + + $migration->up(); +}); diff --git a/tests/Feature/Server/ServerStorageCheckNotificationIntervalTest.php b/tests/Feature/Server/ServerStorageCheckNotificationIntervalTest.php new file mode 100644 index 0000000000..ff0caf1a98 --- /dev/null +++ b/tests/Feature/Server/ServerStorageCheckNotificationIntervalTest.php @@ -0,0 +1,122 @@ + 0]); + Carbon::setTestNow(Carbon::parse('2026-09-29 12:00:00', 'UTC')); +}); + +afterEach(function () { + Carbon::setTestNow(); +}); + +function diskUsageIntervalServer(int $intervalHours = 24): Server +{ + $team = Team::factory()->create(); + $team->emailNotificationSettings()->update([ + 'use_instance_email_settings' => true, + 'server_disk_usage_email_notifications' => true, + ]); + $server = Server::factory()->create(['team_id' => $team->id, 'ip' => '10.0.0.10']); + $server->settings->update([ + 'is_reachable' => true, + 'is_usable' => true, + 'force_disabled' => false, + 'server_disk_usage_notification_threshold' => 80, + 'server_disk_usage_notification_interval_hours' => $intervalHours, + ]); + + return $server->fresh(); +} + +function runDiskUsageCheck(Server $server, int $percentage): void +{ + (new ServerStorageCheckJob($server->fresh(), $percentage))->handle(); +} + +it('sends one high disk usage notification per configured interval', function () { + $server = diskUsageIntervalServer(intervalHours: 6); + + runDiskUsageCheck($server, 93); + Carbon::setTestNow(now()->addMinutes(10)); + runDiskUsageCheck($server, 93); + Carbon::setTestNow(now()->addHours(5)); + runDiskUsageCheck($server, 94); + + Notification::assertSentToTimes($server->team, HighDiskUsage::class, 1); + + Carbon::setTestNow(now()->addMinutes(51)); + runDiskUsageCheck($server, 94); + + Notification::assertSentToTimes($server->team, HighDiskUsage::class, 2); +}); + +it('defaults to one notification per 24 hours', function () { + $team = Team::factory()->create(); + $server = Server::factory()->create(['team_id' => $team->id]); + + expect($server->settings->fresh()->server_disk_usage_notification_interval_hours)->toBe(24); +}); + +it('does not notify below the threshold', function () { + $server = diskUsageIntervalServer(); + + runDiskUsageCheck($server, 50); + + Notification::assertNothingSent(); + expect(NotificationThrottle::wasSent($server, HighDiskUsage::class))->toBeFalse(); +}); + +it('saves the notification interval from the server advanced settings', function () { + $team = Team::factory()->create(); + $user = User::factory()->create(); + $team->members()->attach($user, ['role' => 'owner']); + $server = Server::factory()->create(['team_id' => $team->id]); + + $this->actingAs($user); + session(['currentTeam' => $team]); + + Livewire::test(Advanced::class, ['server_uuid' => $server->uuid]) + ->assertSet('serverDiskUsageNotificationIntervalHours', 24) + ->set('serverDiskUsageNotificationIntervalHours', 0) + ->call('submit') + ->assertHasErrors(['serverDiskUsageNotificationIntervalHours']) + ->set('serverDiskUsageNotificationIntervalHours', 12) + ->call('submit') + ->assertHasNoErrors() + ->assertDispatched('success'); + + expect($server->settings->fresh()->server_disk_usage_notification_interval_hours)->toBe(12); +}); + +it('does not let a member change the notification interval', function () { + $team = Team::factory()->create(); + $user = User::factory()->create(); + $team->members()->attach($user, ['role' => 'member']); + $server = Server::factory()->create(['team_id' => $team->id]); + + $this->actingAs($user); + session(['currentTeam' => $team]); + + Livewire::test(Advanced::class, ['server_uuid' => $server->uuid]) + ->set('serverDiskUsageNotificationIntervalHours', 2) + ->call('submit'); + + expect($server->settings->fresh()->server_disk_usage_notification_interval_hours)->toBe(24); +}); diff --git a/tests/Feature/ServerReachabilityNotificationTest.php b/tests/Feature/ServerReachabilityNotificationTest.php index e996ba0286..4fdd8df86c 100644 --- a/tests/Feature/ServerReachabilityNotificationTest.php +++ b/tests/Feature/ServerReachabilityNotificationTest.php @@ -1,6 +1,7 @@ server->settings()->update(['is_reachable' => false]); $this->server->forceFill([ 'unreachable_count' => 2, - 'unreachable_notification_sent' => false, ])->save(); ServerReachabilityChanged::dispatch($this->server->fresh()); @@ -41,7 +41,6 @@ it('does not send Unreachable on first transient failure (count=1)', function () $this->server->settings()->update(['is_reachable' => false]); $this->server->forceFill([ 'unreachable_count' => 1, - 'unreachable_notification_sent' => false, ])->save(); ServerReachabilityChanged::dispatch($this->server->fresh()); @@ -53,8 +52,8 @@ it('does not send Unreachable when already notified', function () { $this->server->settings()->update(['is_reachable' => false]); $this->server->forceFill([ 'unreachable_count' => 5, - 'unreachable_notification_sent' => true, ])->save(); + NotificationThrottle::record($this->server, Unreachable::class); ServerReachabilityChanged::dispatch($this->server->fresh()); @@ -65,8 +64,8 @@ it('sends Reachable notification on recovery when previously notified', function $this->server->settings()->update(['is_reachable' => true]); $this->server->forceFill([ 'unreachable_count' => 0, - 'unreachable_notification_sent' => true, ])->save(); + NotificationThrottle::record($this->server, Unreachable::class); $fresh = $this->server->fresh(); expect($fresh->unreachable_notification_sent)->toBeTrue(); @@ -82,7 +81,6 @@ it('does not send Reachable when never notified', function () { $this->server->settings()->update(['is_reachable' => true]); $this->server->forceFill([ 'unreachable_count' => 0, - 'unreachable_notification_sent' => false, ])->save(); ServerReachabilityChanged::dispatch($this->server->fresh()); @@ -94,7 +92,6 @@ it('routes Unreachable notification through EmailChannel when email toggle is on $this->server->settings()->update(['is_reachable' => false]); $this->server->forceFill([ 'unreachable_count' => 2, - 'unreachable_notification_sent' => false, ])->save(); ServerReachabilityChanged::dispatch($this->server->fresh()); diff --git a/tests/Feature/ServerUpdatePrivateKeyApiTest.php b/tests/Feature/ServerUpdatePrivateKeyApiTest.php index 7940c0d9a1..ae567d2da7 100644 --- a/tests/Feature/ServerUpdatePrivateKeyApiTest.php +++ b/tests/Feature/ServerUpdatePrivateKeyApiTest.php @@ -227,3 +227,20 @@ it('rejects an invalid disk usage check frequency without partially updating the ->and($server->settings->server_role->value)->toBe('both') ->and($server->proxy->get('type'))->toBe('TRAEFIK'); }); + +it('updates the disk usage notification interval', function () { + patchServerUpdatePrivateKeyApi($this, $this->server, $this->bearerToken, [ + 'server_disk_usage_notification_interval_hours' => 48, + ])->assertCreated(); + + expect($this->server->settings->fresh()->server_disk_usage_notification_interval_hours)->toBe(48); +}); + +it('rejects an out of range disk usage notification interval', function () { + patchServerUpdatePrivateKeyApi($this, $this->server, $this->bearerToken, [ + 'server_disk_usage_notification_interval_hours' => 0, + ])->assertUnprocessable() + ->assertJsonValidationErrors('server_disk_usage_notification_interval_hours'); + + expect($this->server->settings->fresh()->server_disk_usage_notification_interval_hours)->toBe(24); +}); diff --git a/tests/Feature/Subscription/SyncStripeSubscriptionsActionTest.php b/tests/Feature/Subscription/SyncStripeSubscriptionsActionTest.php index 31070053d6..a59ae659de 100644 --- a/tests/Feature/Subscription/SyncStripeSubscriptionsActionTest.php +++ b/tests/Feature/Subscription/SyncStripeSubscriptionsActionTest.php @@ -42,7 +42,6 @@ test('fix mode fully ends a locally active subscription with an invalid Stripe s $server = Server::factory()->create([ 'team_id' => $this->team->id, 'unreachable_count' => 0, - 'unreachable_notification_sent' => false, ]); $stripe = Mockery::mock(StripeClient::class); diff --git a/tests/Feature/ThrottledNotificationTest.php b/tests/Feature/ThrottledNotificationTest.php new file mode 100644 index 0000000000..1528b6ce55 --- /dev/null +++ b/tests/Feature/ThrottledNotificationTest.php @@ -0,0 +1,102 @@ +team = Team::factory()->create(); + $this->team->emailNotificationSettings()->update([ + 'use_instance_email_settings' => true, + 'status_change_email_notifications' => true, + ]); + $this->server = Server::factory()->create(['team_id' => $this->team->id]); +}); + +afterEach(function () { + Carbon::setTestNow(); +}); + +it('sends a throttled notification once per interval for the same resource', function () { + $otherServer = Server::factory()->create(['team_id' => $this->team->id]); + + $this->team->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server)); + $this->team->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server)); + $this->team->notify(new ContainerRestarted('coolify-proxy', $otherServer, restartedResource: $otherServer)); + + Notification::assertSentToTimes($this->team, ContainerRestarted::class, 2); + + Carbon::setTestNow(now()->addMinutes(59)); + $this->team->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server)); + Notification::assertSentToTimes($this->team, ContainerRestarted::class, 2); + + Carbon::setTestNow(now()->addMinutes(2)); + $this->team->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server)); + Notification::assertSentToTimes($this->team, ContainerRestarted::class, 3); +}); + +it('does not throttle a notification without a throttle subject', function () { + $this->team->notify(new ContainerRestarted('TCP Proxy for db has been disabled', $this->server)); + $this->team->notify(new ContainerRestarted('TCP Proxy for db has been disabled', $this->server)); + + Notification::assertSentToTimes($this->team, ContainerRestarted::class, 2); +}); + +it('sends one TCP proxy restart notification per hour while the proxy does not stay up', function () { + $project = Project::factory()->create(['team_id' => $this->team->id]); + $environment = Environment::factory()->create(['project_id' => $project->id]); + $destination = $this->server->standaloneDockers()->firstOrFail(); + $database = StandalonePostgresql::create([ + 'name' => 'public-db', + 'postgres_user' => 'postgres', + 'postgres_password' => encrypt('password'), + 'postgres_db' => 'app', + 'image' => 'postgres:16-alpine', + 'is_public' => true, + 'public_port' => 54321, + 'environment_id' => $environment->id, + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + ]); + + // The database runs, but its "-proxy" container is missing on every push. + $push = fn () => (new PushServerUpdateJob($this->server, [ + 'containers' => [[ + 'name' => $database->uuid, + 'state' => 'running', + 'health_status' => 'healthy', + 'labels' => ['coolify.managed' => true, 'com.docker.compose.service' => $database->uuid], + ]], + 'filesystem_usage_root' => ['used_percentage' => 10], + ]))->handle(); + + $push(); + Carbon::setTestNow(now()->addMinute()); + $push(); + Carbon::setTestNow(now()->addMinute()); + $push(); + + Notification::assertSentToTimes($this->team, ContainerRestarted::class, 1); + + Carbon::setTestNow(now()->addHour()); + $push(); + + Notification::assertSentToTimes($this->team, ContainerRestarted::class, 2); +});