feat(s3): revalidate unusable S3 storages hourly

Re-test S3 storages marked unusable every hour so a storage that becomes
reachable again is usable without manual action. Failures keep the
storage unusable and reuse the existing once-per-outage email.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/Console/Kernel.php

@@ -12,6 +12,7 @@ use App\Jobs\CleanupStaleMultiplexedConnections;
 use App\Jobs\PullChangelog;
 use App\Jobs\PullTemplatesFromCDN;
 use App\Jobs\RegenerateSslCertJob;
+use App\Jobs\RevalidateUnusableS3StoragesJob;
 use App\Jobs\ServerManagerJob;
 use App\Jobs\UpdateCoolifyJob;
 use App\Models\InstanceSettings;
@@ -63,6 +64,7 @@ class Kernel extends ConsoleKernel
             ->runInBackground();
         $this->scheduleInstance->job(new ApiTokenExpirationWarningJob)->hourly()->onOneServer();
         $this->scheduleInstance->job(new CheckMissingDatabaseBackupsJob)->hourly()->onOneServer();
+        $this->scheduleInstance->job(new RevalidateUnusableS3StoragesJob)->hourly()->onOneServer();
 
         if (isDev()) {
             // Instance Jobs

app/Jobs/RevalidateUnusableS3StoragesJob.php

@@ -0,0 +1,39 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Models\S3Storage;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeEncrypted;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+
+/**
+ * Re-tests S3 storages that failed validation, so a storage that becomes reachable again is usable without manual action.
+ */
+class RevalidateUnusableS3StoragesJob implements ShouldBeEncrypted, ShouldBeUnique, ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public $tries = 1;
+
+    public $timeout = 1800;
+
+    public function handle(): void
+    {
+        S3Storage::query()
+            ->where('is_usable', false)
+            ->chunkById(100, function ($storages): void {
+                foreach ($storages as $storage) {
+                    try {
+                        $storage->testConnection(shouldSave: true);
+                    } catch (\Throwable) {
+                        // testConnection() keeps the storage unusable and notifies the team once.
+                    }
+                }
+            });
+    }
+}

tests/Feature/RevalidateUnusableS3StoragesJobTest.php

@@ -0,0 +1,63 @@
+<?php
+
+use App\Jobs\RevalidateUnusableS3StoragesJob;
+use App\Models\S3Storage;
+use App\Models\Team;
+use Illuminate\Filesystem\FilesystemAdapter;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Storage;
+
+uses(RefreshDatabase::class);
+
+function revalidationStorage(Team $team, array $attributes = []): S3Storage
+{
+    return S3Storage::create(array_merge([
+        'name' => 'Storage',
+        'region' => 'us-east-1',
+        'key' => 'key',
+        'secret' => 'secret',
+        'bucket' => 'bucket',
+        'endpoint' => 'https://93.184.216.34',
+        'team_id' => $team->id,
+        'is_usable' => false,
+        'unusable_email_sent' => true,
+    ], $attributes));
+}
+
+it('marks an unusable storage as usable when the connection works again', function () {
+    $storage = revalidationStorage(Team::factory()->create());
+    $disk = Mockery::mock(FilesystemAdapter::class);
+    $disk->shouldReceive('files')->once()->andReturn([]);
+    Storage::shouldReceive('build')->once()->andReturn($disk);
+
+    (new RevalidateUnusableS3StoragesJob)->handle();
+
+    $storage->refresh();
+    expect($storage->is_usable)->toBeTrue()
+        ->and($storage->unusable_email_sent)->toBeFalsy();
+});
+
+it('keeps a storage unusable when the connection still fails and continues with the next one', function () {
+    $team = Team::factory()->create();
+    $failing = revalidationStorage($team, ['bucket' => 'failing-bucket']);
+    $working = revalidationStorage($team, ['bucket' => 'working-bucket']);
+    $failingDisk = Mockery::mock(FilesystemAdapter::class);
+    $failingDisk->shouldReceive('files')->once()->andThrow(new RuntimeException('Access denied'));
+    $workingDisk = Mockery::mock(FilesystemAdapter::class);
+    $workingDisk->shouldReceive('files')->once()->andReturn([]);
+    Storage::shouldReceive('build')->twice()->andReturnUsing(
+        fn (array $config) => $config['bucket'] === 'failing-bucket' ? $failingDisk : $workingDisk
+    );
+
+    (new RevalidateUnusableS3StoragesJob)->handle();
+
+    expect($failing->fresh()->is_usable)->toBeFalse()
+        ->and($working->fresh()->is_usable)->toBeTrue();
+});
+
+it('does not test storages that are already usable', function () {
+    revalidationStorage(Team::factory()->create(), ['is_usable' => true]);
+    Storage::shouldReceive('build')->never();
+
+    (new RevalidateUnusableS3StoragesJob)->handle();
+});