Merge remote-tracking branch 'origin/next' into compose-settings-memory-crash

2026-09-28 02:06:37 -04:00 · 2026-07-16 16:10:11 +02:00
parent 60129c2c47 7d699818e8
commit 5ce61ff1a1
38 changed files with 1627 additions and 134 deletions
@@ -0,0 +1,47 @@
+<?php
+
+use App\Livewire\Admin\Index as AdminIndex;
+use App\Models\InstanceSettings;
+use App\Models\Subscription;
+use App\Models\Team;
+use App\Models\User;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Livewire\Livewire;
+
+uses(RefreshDatabase::class);
+
+test('admin search only shows users with paid subscriptions as active', function () {
+    config()->set('cache.default', 'array');
+    config()->set('constants.coolify.self_hosted', false);
+
+    InstanceSettings::unguarded(
+        fn () => InstanceSettings::query()->firstOrCreate(['id' => 0])
+    );
+
+    $rootTeam = Team::find(0) ?? Team::factory()->create(['id' => 0]);
+    $rootUser = User::find(0) ?? User::factory()->create(['id' => 0]);
+    $rootTeam->members()->syncWithoutDetaching([
+        $rootUser->id => ['role' => 'admin'],
+    ]);
+
+    $inactiveUser = User::factory()->create(['email' => 'inactive@example.com']);
+    $inactiveTeam = Team::factory()->create();
+    $inactiveTeam->members()->attach($inactiveUser->id, ['role' => 'owner']);
+    Subscription::create([
+        'team_id' => $inactiveTeam->id,
+        'stripe_subscription_id' => 'sub_stale',
+        'stripe_invoice_paid' => false,
+    ]);
+
+    $this->actingAs($rootUser);
+    session(['currentTeam' => ['id' => $rootTeam->id]]);
+
+    Livewire::test(AdminIndex::class)
+        ->set([
+            'foundUsers' => collect(),
+            'search' => 'inactive@example.com',
+        ])
+        ->call('submitSearch')
+        ->assertSee('No')
+        ->assertDontSee('Yes');
+});
@@ -5,8 +5,10 @@ use App\Models\Project;
 use App\Models\S3Storage;
 use App\Models\ScheduledDatabaseBackup;
 use App\Models\Server;
+use App\Models\StandaloneClickhouse;
 use App\Models\StandaloneDocker;
 use App\Models\StandalonePostgresql;
+use App\Models\StandaloneRedis;
 use App\Models\Team;
 use App\Models\User;
 use Illuminate\Foundation\Testing\RefreshDatabase;
@@ -15,7 +17,7 @@ use Illuminate\Support\Str;
 uses(RefreshDatabase::class);

 beforeEach(function () {
-    InstanceSettings::updateOrCreate(['id' => 0]);
+    InstanceSettings::forceCreate(['id' => 0, 'is_api_enabled' => true]);

    $this->team = Team::factory()->create();
    $this->user = User::factory()->create();
@@ -77,6 +79,56 @@ function backupHeaders(): array
 }

 describe('POST /api/v1/databases/{uuid}/backups', function () {
+    test('rejects backup configurations for unsupported database types', function () {
+        $database = StandaloneRedis::create([
+            'uuid' => (string) Str::uuid(),
+            'name' => 'Redis DB',
+            'status' => 'running',
+            'environment_id' => $this->environment->id,
+            'destination_id' => $this->destination->id,
+            'destination_type' => $this->destination->getMorphClass(),
+        ]);
+
+        $response = $this->withHeaders(backupHeaders())
+            ->postJson("/api/v1/databases/{$database->uuid}/backups", [
+                'frequency' => 'daily',
+            ]);
+
+        $response->assertUnprocessable()
+            ->assertJson([
+                'message' => 'Scheduled backups are not supported for this database type.',
+            ]);
+
+        expect(ScheduledDatabaseBackup::count())->toBe(0);
+    });
+
+    test('defaults clickhouse backups to its configured database', function () {
+        $database = StandaloneClickhouse::create([
+            'uuid' => (string) Str::uuid(),
+            'name' => 'ClickHouse DB',
+            'clickhouse_admin_user' => 'default',
+            'clickhouse_admin_password' => 'password',
+            'clickhouse_db' => 'analytics',
+            'image' => 'clickhouse/clickhouse-server:25.11',
+            'status' => 'running',
+            'environment_id' => $this->environment->id,
+            'destination_id' => $this->destination->id,
+            'destination_type' => $this->destination->getMorphClass(),
+        ]);
+
+        $response = $this->withHeaders(backupHeaders())
+            ->postJson("/api/v1/databases/{$database->uuid}/backups", [
+                'frequency' => 'daily',
+            ]);
+
+        $response->assertCreated();
+
+        $backup = ScheduledDatabaseBackup::where('uuid', $response->json('uuid'))->firstOrFail();
+
+        expect($backup->databases_to_backup)->toBe('analytics')
+            ->and($backup->database_type)->toBe(StandaloneClickhouse::class);
+    });
+
    test('creates backup configuration with valid frequency', function () {
        $response = $this->withHeaders(backupHeaders())
            ->postJson("/api/v1/databases/{$this->database->uuid}/backups", [
@@ -0,0 +1,154 @@
+<?php
+
+use App\Models\Application;
+use App\Models\Environment;
+use App\Models\Project;
+use App\Models\Server;
+use App\Models\Subscription;
+use App\Models\Team;
+use App\Models\User;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Artisan;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+    config()->set('constants.coolify.self_hosted', false);
+});
+
+test('it previews eligible unverified users without deleting them', function () {
+    $user = User::factory()->unverified()->create([
+        'email' => 'unverified@example.com',
+    ]);
+
+    $this->artisan('cloud:cleanup-unverified-users')
+        ->expectsOutput('Found 1 unverified user eligible for deletion.')
+        ->expectsOutput('Dry run only. Use --yes to delete eligible users.')
+        ->assertSuccessful();
+
+    $this->assertModelExists($user);
+});
+
+test('it reports dry run progress', function () {
+    $users = User::factory()->unverified()->count(2)->create();
+
+    $exitCode = Artisan::call('cloud:cleanup-unverified-users');
+
+    expect($exitCode)->toBe(0)
+        ->and(Artisan::output())->toContain('Checking eligible users: 2/2');
+
+    $users->each(fn (User $user) => $this->assertModelExists($user));
+});
+
+test('it deletes eligible unverified users with the yes option', function () {
+    $user = User::factory()->unverified()->create([
+        'email' => 'unverified@example.com',
+    ]);
+
+    $this->artisan('cloud:cleanup-unverified-users', ['--yes' => true])
+        ->expectsOutput('Deleted 1 unverified user.')
+        ->assertSuccessful();
+
+    $this->assertModelMissing($user);
+});
+
+test('it reports deletion progress', function () {
+    User::factory()->unverified()->count(2)->create();
+
+    $exitCode = Artisan::call('cloud:cleanup-unverified-users', ['--yes' => true]);
+
+    expect($exitCode)->toBe(0)
+        ->and(Artisan::output())->toContain('Deleting eligible users: 2/2');
+});
+
+test('it keeps verified users', function () {
+    $user = User::factory()->create([
+        'email' => 'verified@example.com',
+    ]);
+
+    $this->artisan('cloud:cleanup-unverified-users', ['--yes' => true])
+        ->expectsOutput('Deleted 0 unverified users.')
+        ->assertSuccessful();
+
+    $this->assertModelExists($user);
+});
+
+test('it keeps unverified users with a Stripe subscription record', function () {
+    $user = User::factory()->unverified()->create([
+        'email' => 'subscribed@example.com',
+    ]);
+
+    Subscription::create([
+        'team_id' => $user->teams()->firstOrFail()->id,
+        'stripe_invoice_paid' => false,
+    ]);
+
+    $this->artisan('cloud:cleanup-unverified-users', ['--yes' => true])
+        ->expectsOutput('Deleted 0 unverified users.')
+        ->assertSuccessful();
+
+    $this->assertModelExists($user);
+});
+
+test('it keeps unverified users with defined resources', function () {
+    $user = User::factory()->unverified()->create([
+        'email' => 'resource-owner@example.com',
+    ]);
+
+    $project = Project::factory()->create([
+        'team_id' => $user->teams()->firstOrFail()->id,
+    ]);
+    $environment = Environment::factory()->create([
+        'project_id' => $project->id,
+    ]);
+    Application::factory()->create([
+        'environment_id' => $environment->id,
+    ]);
+
+    $this->artisan('cloud:cleanup-unverified-users', ['--yes' => true])
+        ->expectsOutput('Deleted 0 unverified users.')
+        ->assertSuccessful();
+
+    $this->assertModelExists($user);
+});
+
+test('it keeps unverified users with servers', function () {
+    $user = User::factory()->unverified()->create([
+        'email' => 'server-owner@example.com',
+    ]);
+
+    Server::factory()->create([
+        'team_id' => $user->teams()->firstOrFail()->id,
+    ]);
+
+    $this->artisan('cloud:cleanup-unverified-users', ['--yes' => true])
+        ->expectsOutput('Deleted 0 unverified users.')
+        ->assertSuccessful();
+
+    $this->assertModelExists($user);
+});
+
+test('it keeps unverified root team members', function () {
+    $user = User::factory()->unverified()->create([
+        'email' => 'root-member@example.com',
+    ]);
+    $rootTeam = Team::factory()->create([
+        'id' => 0,
+        'name' => 'Root Team',
+    ]);
+    $rootTeam->members()->attach($user->id, ['role' => 'admin']);
+
+    $this->artisan('cloud:cleanup-unverified-users', ['--yes' => true])
+        ->expectsOutput('Deleted 0 unverified users.')
+        ->assertSuccessful();
+
+    $this->assertModelExists($user);
+});
+
+test('it only runs on Coolify Cloud', function () {
+    config()->set('constants.coolify.self_hosted', true);
+
+    $this->artisan('cloud:cleanup-unverified-users')
+        ->expectsOutput('This command can only be run on Coolify Cloud.')
+        ->assertFailed();
+});
@@ -0,0 +1,101 @@
+<?php
+
+use App\Models\Subscription;
+use App\Models\User;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Storage;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+    Storage::fake('backups');
+    config()->set('constants.coolify.self_hosted', false);
+});
+
+test('it exports subscribed and unsubscribed verified users to separate files without tags', function () {
+    User::factory()->create([
+        'id' => 0,
+        'name' => 'Root User',
+        'email' => 'root@example.com',
+    ]);
+
+    $subscribedUser = User::factory()->create([
+        'name' => 'Ada Lovelace',
+        'email' => 'ada@example.com',
+    ]);
+
+    Subscription::create([
+        'team_id' => $subscribedUser->teams()->firstOrFail()->id,
+        'stripe_invoice_paid' => true,
+        'stripe_subscription_id' => 'sub_active',
+    ]);
+
+    User::factory()->create([
+        'name' => 'Grace',
+        'email' => 'grace@example.com',
+    ]);
+
+    User::factory()->unverified()->create([
+        'name' => 'Unverified User',
+        'email' => 'unverified@example.com',
+    ]);
+
+    Storage::disk('backups')->put('cloud-users.csv', 'old export');
+
+    $this->artisan('cloud:export-users')->assertSuccessful();
+
+    Storage::disk('backups')->assertExists([
+        'cloud-users-subscribed.csv',
+        'cloud-users-unsubscribed.csv',
+    ]);
+    Storage::disk('backups')->assertMissing('cloud-users.csv');
+
+    $readCsv = function (string $filename): array {
+        $output = fopen(Storage::disk('backups')->path($filename), 'rb');
+        $rows = [];
+
+        while (($row = fgetcsv($output, null, ',', '"', '')) !== false) {
+            $rows[] = $row;
+        }
+
+        fclose($output);
+
+        return $rows;
+    };
+
+    $header = [
+        'email',
+        'first_name',
+        'last_name',
+        'lifetime_value_currency',
+        'lifetime_value_amount',
+        'utm_campaign',
+        'utm_source',
+        'utm_medium',
+        'utm_content',
+        'utm_term',
+        'phone',
+    ];
+
+    expect($readCsv('cloud-users-subscribed.csv'))->toBe([
+        $header,
+        ['ada@example.com', 'Ada', 'Lovelace', '', '', '', '', '', '', '', ''],
+    ])->and($readCsv('cloud-users-unsubscribed.csv'))->toBe([
+        $header,
+        ['grace@example.com', 'Grace', '', '', '', '', '', '', '', '', ''],
+    ]);
+});
+
+test('it only runs on Coolify Cloud', function () {
+    config()->set('constants.coolify.self_hosted', true);
+
+    $this->artisan('cloud:export-users')
+        ->expectsOutput('This command can only be run on Coolify Cloud.')
+        ->assertFailed();
+
+    Storage::disk('backups')->assertMissing([
+        'cloud-users.csv',
+        'cloud-users-subscribed.csv',
+        'cloud-users-unsubscribed.csv',
+    ]);
+});
@@ -6,8 +6,10 @@ use App\Models\Project;
 use App\Models\S3Storage;
 use App\Models\ScheduledDatabaseBackup;
 use App\Models\Server;
+use App\Models\StandaloneClickhouse;
 use App\Models\StandaloneDocker;
 use App\Models\StandalonePostgresql;
+use App\Models\StandaloneRedis;
 use App\Models\Team;
 use App\Models\User;
 use Illuminate\Foundation\Testing\RefreshDatabase;
@@ -136,3 +138,49 @@ it('creates a scheduled backup with a valid team-owned S3 storage', function ()
    expect($backup->save_s3)->toBeTruthy();
    expect($backup->s3_storage_id)->toBe($s3->id);
 });
+
+it('creates a clickhouse backup for its configured database', function () {
+    $server = Server::factory()->create(['team_id' => $this->team->id]);
+    $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();
+    $project = Project::factory()->create(['team_id' => $this->team->id]);
+    $environment = Environment::factory()->create(['project_id' => $project->id]);
+    $database = StandaloneClickhouse::create([
+        'name' => 'clickhouse-scheduled-backup',
+        'clickhouse_admin_user' => 'default',
+        'clickhouse_admin_password' => 'password',
+        'clickhouse_db' => 'analytics',
+        'environment_id' => $environment->id,
+        'destination_id' => $destination->id,
+        'destination_type' => $destination->getMorphClass(),
+    ]);
+
+    Livewire::test(CreateScheduledBackup::class, ['database' => $database])
+        ->set('frequency', 'daily')
+        ->call('submit')
+        ->assertDispatched('refreshScheduledBackups');
+
+    $backup = ScheduledDatabaseBackup::firstOrFail();
+
+    expect($backup->database_type)->toBe(StandaloneClickhouse::class)
+        ->and($backup->databases_to_backup)->toBe('analytics');
+});
+
+it('rejects scheduled backups for unsupported database types', function () {
+    $server = Server::factory()->create(['team_id' => $this->team->id]);
+    $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();
+    $project = Project::factory()->create(['team_id' => $this->team->id]);
+    $environment = Environment::factory()->create(['project_id' => $project->id]);
+    $database = StandaloneRedis::create([
+        'name' => 'redis-without-backups',
+        'environment_id' => $environment->id,
+        'destination_id' => $destination->id,
+        'destination_type' => $destination->getMorphClass(),
+    ]);
+
+    Livewire::test(CreateScheduledBackup::class, ['database' => $database])
+        ->set('frequency', 'daily')
+        ->call('submit')
+        ->assertDispatched('error');
+
+    expect(ScheduledDatabaseBackup::count())->toBe(0);
+});
@@ -27,9 +27,9 @@ it('rate limits repeated forgot password attempts from the same ip', function ()

 it('rate limits dotted plus-address forgot password variants of the same email identity across ips', function () {
    $emails = [
-        'ke.vinmcfadden+one@btinternet.com',
-        'kevin.mcfadden+two@btinternet.com',
-        'k.e.v.i.n.m.c.f.a.d.d.e.n+three@btinternet.com',
+        'ke.vinmcfadden+one@gmail.com',
+        'kevin.mcfadden+two@gmail.com',
+        'k.e.v.i.n.m.c.f.a.d.d.e.n+three@gmail.com',
    ];

    foreach ($emails as $index => $email) {
@@ -42,7 +42,24 @@ it('rate limits dotted plus-address forgot password variants of the same email i

    $this->withServerVariables(['REMOTE_ADDR' => '203.0.113.99'])
        ->post('/forgot-password', [
-            'email' => 'k.evin.mcfadden+four@btinternet.com',
+            'email' => 'k.evin.mcfadden+four@gmail.com',
        ])
        ->assertTooManyRequests();
 });
+
+it('keeps distinct dotted and plus-addressed mailboxes in separate forgot password buckets on ordinary domains', function () {
+    $emails = [
+        'john.smith@example.com',
+        'johnsmith@example.com',
+        'johnsmith+one@example.com',
+        'johnsmith+two@example.com',
+    ];
+
+    foreach ($emails as $index => $email) {
+        $this->withServerVariables(['REMOTE_ADDR' => '203.0.113.'.($index + 120)])
+            ->post('/forgot-password', [
+                'email' => $email,
+            ])
+            ->assertSessionHasNoErrors();
+    }
+});
@@ -3,6 +3,7 @@
 use App\Models\InstanceSettings;
 use App\Models\User;
 use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\RateLimiter;

 uses(RefreshDatabase::class);

@@ -42,9 +43,9 @@ it('rate limits repeated registration attempts from the same ip', function () {

 it('rate limits dotted plus-address variants of the same email identity across ips', function () {
    $emails = [
-        'ke.vinmcfadden+one@btinternet.com',
-        'kevin.mcfadden+two@btinternet.com',
-        'k.e.v.i.n.m.c.f.a.d.d.e.n+three@btinternet.com',
+        'ke.vinmcfadden+one@gmail.com',
+        'kevin.mcfadden+two@gmail.com',
+        'k.e.v.i.n.m.c.f.a.d.d.e.n+three@gmail.com',
    ];

    foreach ($emails as $index => $email) {
@@ -64,9 +65,33 @@ it('rate limits dotted plus-address variants of the same email identity across i
    $this->withServerVariables(['REMOTE_ADDR' => '203.0.113.99'])
        ->post('/register', [
            'name' => 'Blocked User',
-            'email' => 'k.evin.mcfadden+four@btinternet.com',
+            'email' => 'k.evin.mcfadden+four@gmail.com',
            'password' => 'Password1!@',
            'password_confirmation' => 'Password1!@',
        ])
        ->assertTooManyRequests();
 });
+
+it('keeps distinct dotted and plus-addressed mailboxes in separate rate limit buckets on ordinary domains', function () {
+    $registrationIpKey = 'registration:ip:'.sha1('127.0.0.1');
+    $emails = [
+        'john.smith@example.com',
+        'johnsmith@example.com',
+        'johnsmith+one@example.com',
+        'johnsmith+two@example.com',
+    ];
+
+    foreach ($emails as $index => $email) {
+        $this->post('/register', [
+            'name' => "Distinct User {$index}",
+            'email' => $email,
+            'password' => 'Password1!@',
+            'password_confirmation' => 'Password1!@',
+        ])
+            ->assertRedirect();
+
+        auth()->logout();
+        $this->flushSession();
+        RateLimiter::clear($registrationIpKey);
+    }
+});
@@ -48,3 +48,13 @@ it('schedules every production job with onOneServer', function () {
        );
    });
 });
+
+it('does not schedule Stripe subscription reconciliation automatically', function () {
+    $schedule = app(Schedule::class);
+
+    $event = collect($schedule->events())->first(
+        fn ($event) => str_contains((string) $event->description, 'SyncStripeSubscriptions')
+    );
+
+    expect($event)->toBeNull();
+});
@@ -0,0 +1,296 @@
+<?php
+
+use App\Actions\Stripe\SyncStripeSubscriptions;
+use App\Models\Server;
+use App\Models\Subscription;
+use App\Models\Team;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Stripe\Collection as StripeCollection;
+use Stripe\Service\CustomerService;
+use Stripe\Service\SubscriptionService;
+use Stripe\StripeClient;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+    config()->set('constants.coolify.self_hosted', false);
+    config()->set('subscription.provider', 'stripe');
+
+    $this->team = Team::factory()->create();
+    $this->subscription = Subscription::create([
+        'team_id' => $this->team->id,
+        'stripe_subscription_id' => 'sub_stale',
+        'stripe_customer_id' => 'cus_stale',
+        'stripe_invoice_paid' => true,
+    ]);
+});
+
+afterEach(function () {
+    SyncStripeSubscriptions::clearFake();
+});
+
+function stripeSubscriptionCollection(array $subscriptions = []): StripeCollection
+{
+    return StripeCollection::constructFrom([
+        'data' => $subscriptions,
+        'has_more' => false,
+        'url' => '/v1/subscriptions',
+    ]);
+}
+
+test('fix mode fully ends a locally active subscription with an invalid Stripe status', function () {
+    $server = Server::factory()->create([
+        'team_id' => $this->team->id,
+        'unreachable_count' => 0,
+        'unreachable_notification_sent' => false,
+    ]);
+
+    $stripe = Mockery::mock(StripeClient::class);
+    $subscriptions = Mockery::mock(SubscriptionService::class);
+    $customers = Mockery::mock(CustomerService::class);
+    $stripe->subscriptions = $subscriptions;
+    $stripe->customers = $customers;
+
+    $subscriptions->shouldReceive('all')->twice()->andReturn(stripeSubscriptionCollection());
+    $subscriptions->shouldReceive('retrieve')->with('sub_stale')->andReturn((object) [
+        'status' => 'unpaid',
+        'customer' => 'cus_stale',
+    ]);
+    $customers->shouldReceive('retrieve')->with('cus_stale')->andThrow(new RuntimeException('Customer unavailable'));
+
+    $this->instance(StripeClient::class, $stripe);
+
+    $progress = [];
+
+    SyncStripeSubscriptions::run(
+        fix: true,
+        onProgress: function (string $stage, int $current, ?int $total) use (&$progress): void {
+            $progress[] = [$stage, $current, $total];
+        },
+    );
+
+    $this->subscription->refresh();
+    $server->refresh();
+
+    expect($this->subscription->stripe_invoice_paid)->toBeFalsy()
+        ->and($this->subscription->stripe_subscription_id)->toBeNull()
+        ->and($server->unreachable_count)->toBe(3)
+        ->and($server->unreachable_notification_sent)->toBeTrue()
+        ->and($progress)->toContain(['checking', 0, 1])
+        ->and($progress)->toContain(['checking', 1, 1]);
+});
+
+test('a same-email subscription not linked to the team is left for manual review', function () {
+    $stripe = Mockery::mock(StripeClient::class);
+    $subscriptions = Mockery::mock(SubscriptionService::class);
+    $customers = Mockery::mock(CustomerService::class);
+    $stripe->subscriptions = $subscriptions;
+    $stripe->customers = $customers;
+
+    $subscriptions->shouldReceive('all')
+        ->with(Mockery::on(fn (array $parameters) => isset($parameters['status'])))
+        ->twice()
+        ->andReturn(stripeSubscriptionCollection());
+    $subscriptions->shouldReceive('retrieve')->with('sub_stale')->andReturn((object) [
+        'status' => 'canceled',
+        'customer' => 'cus_stale',
+    ]);
+    $customers->shouldReceive('retrieve')->with('cus_stale')->andReturn((object) [
+        'email' => 'customer@example.com',
+    ]);
+    $customers->shouldReceive('all')->andReturn((object) [
+        'data' => [(object) ['id' => 'cus_active']],
+    ]);
+    $subscriptions->shouldReceive('all')->with([
+        'customer' => 'cus_active',
+        'limit' => 10,
+    ])->andReturn((object) [
+        'data' => [(object) [
+            'id' => 'sub_active_elsewhere',
+            'status' => 'active',
+        ]],
+    ]);
+
+    $this->instance(StripeClient::class, $stripe);
+
+    $result = SyncStripeSubscriptions::run(fix: true);
+
+    $this->subscription->refresh();
+
+    expect($result['resubscribed'])->toHaveCount(1)
+        ->and($result['discrepancies'])->toHaveCount(1)
+        ->and($result['discrepancies'][0]['resolution'])->toBe('manual_review')
+        ->and($this->subscription->stripe_invoice_paid)->toBeTruthy()
+        ->and($this->subscription->stripe_subscription_id)->toBe('sub_stale');
+});
+
+test('a stale local subscription is deleted when its team has a valid replacement', function () {
+    $replacement = Subscription::create([
+        'team_id' => $this->team->id,
+        'stripe_subscription_id' => 'sub_active_elsewhere',
+        'stripe_customer_id' => 'cus_active',
+        'stripe_invoice_paid' => true,
+    ]);
+    $server = Server::factory()->create(['team_id' => $this->team->id]);
+    $server->settings->update([
+        'is_reachable' => true,
+        'is_usable' => true,
+    ]);
+
+    $stripe = Mockery::mock(StripeClient::class);
+    $subscriptions = Mockery::mock(SubscriptionService::class);
+    $customers = Mockery::mock(CustomerService::class);
+    $stripe->subscriptions = $subscriptions;
+    $stripe->customers = $customers;
+
+    $subscriptions->shouldReceive('all')
+        ->with(['status' => 'active', 'limit' => 100])
+        ->andReturn(stripeSubscriptionCollection([
+            ['id' => 'sub_active_elsewhere'],
+        ]));
+    $subscriptions->shouldReceive('all')
+        ->with(['status' => 'past_due', 'limit' => 100])
+        ->andReturn(stripeSubscriptionCollection());
+    $subscriptions->shouldReceive('retrieve')->with('sub_stale')->andReturn((object) [
+        'status' => 'canceled',
+        'customer' => 'cus_stale',
+    ]);
+    $customers->shouldReceive('retrieve')->with('cus_stale')->andReturn((object) [
+        'email' => 'customer@example.com',
+    ]);
+    $customers->shouldReceive('all')->andReturn((object) [
+        'data' => [(object) ['id' => 'cus_active']],
+    ]);
+    $subscriptions->shouldReceive('all')->with([
+        'customer' => 'cus_active',
+        'limit' => 10,
+    ])->andReturn((object) [
+        'data' => [(object) [
+            'id' => 'sub_active_elsewhere',
+            'status' => 'active',
+        ]],
+    ]);
+
+    $this->instance(StripeClient::class, $stripe);
+
+    $result = SyncStripeSubscriptions::run(fix: true);
+
+    expect($result['discrepancies'][0]['resolution'])->toBe('delete_stale')
+        ->and($this->subscription->fresh())->toBeNull()
+        ->and($replacement->fresh())->not->toBeNull()
+        ->and($replacement->fresh()->stripe_invoice_paid)->toBeTruthy()
+        ->and($server->settings->fresh()->is_reachable)->toBeTruthy()
+        ->and($server->settings->fresh()->is_usable)->toBeTruthy();
+});
+
+test('a stale duplicate is deleted when its team already has an inactive subscription row', function () {
+    $inactiveSubscription = Subscription::create([
+        'team_id' => $this->team->id,
+        'stripe_customer_id' => 'cus_stale',
+        'stripe_invoice_paid' => false,
+    ]);
+    $server = Server::factory()->create(['team_id' => $this->team->id]);
+    $server->settings->update([
+        'is_reachable' => true,
+        'is_usable' => true,
+    ]);
+
+    $stripe = Mockery::mock(StripeClient::class);
+    $subscriptions = Mockery::mock(SubscriptionService::class);
+    $customers = Mockery::mock(CustomerService::class);
+    $stripe->subscriptions = $subscriptions;
+    $stripe->customers = $customers;
+
+    $subscriptions->shouldReceive('all')->twice()->andReturn(stripeSubscriptionCollection());
+    $subscriptions->shouldReceive('retrieve')->with('sub_stale')->andReturn((object) [
+        'status' => 'canceled',
+        'customer' => 'cus_stale',
+    ]);
+    $customers->shouldReceive('retrieve')->with('cus_stale')->andThrow(new RuntimeException('Customer unavailable'));
+
+    $this->instance(StripeClient::class, $stripe);
+
+    $result = SyncStripeSubscriptions::run(fix: true);
+
+    expect($result['discrepancies'][0]['resolution'])->toBe('delete_stale')
+        ->and($this->subscription->fresh())->toBeNull()
+        ->and($inactiveSubscription->fresh())->not->toBeNull()
+        ->and($server->settings->fresh()->is_reachable)->toBeFalsy()
+        ->and($server->settings->fresh()->is_usable)->toBeFalsy();
+});
+
+test('valid Stripe subscriptions remain active', function (string $status) {
+    $stripe = Mockery::mock(StripeClient::class);
+    $subscriptions = Mockery::mock(SubscriptionService::class);
+    $stripe->subscriptions = $subscriptions;
+
+    $subscriptions->shouldReceive('all')
+        ->with(['status' => $status, 'limit' => 100])
+        ->andReturn(stripeSubscriptionCollection([
+            ['id' => 'sub_stale'],
+        ]));
+    $subscriptions->shouldReceive('all')
+        ->with(['status' => $status === 'active' ? 'past_due' : 'active', 'limit' => 100])
+        ->andReturn(stripeSubscriptionCollection());
+    $subscriptions->shouldNotReceive('retrieve');
+
+    $this->instance(StripeClient::class, $stripe);
+
+    $result = SyncStripeSubscriptions::run(fix: true);
+
+    $this->subscription->refresh();
+
+    expect($result['discrepancies'])->toBeEmpty()
+        ->and($this->subscription->stripe_invoice_paid)->toBeTruthy()
+        ->and($this->subscription->stripe_subscription_id)->toBe('sub_stale');
+})->with(['active', 'past_due']);
+
+test('the terminal command runs the reconciliation action synchronously', function () {
+    SyncStripeSubscriptions::shouldRun()
+        ->once()
+        ->withArgs(fn (bool $fix, ?Closure $onProgress) => $fix === false && $onProgress instanceof Closure)
+        ->andReturnUsing(function (bool $fix, Closure $onProgress): array {
+            $onProgress('checking', 2, 10);
+
+            return [
+                'total_checked' => 0,
+                'discrepancies' => [],
+                'resubscribed' => [],
+                'errors' => [],
+                'fixed' => false,
+            ];
+        });
+
+    $this->artisan('cloud:sync-stripe-subscriptions')
+        ->expectsOutputToContain('Checking stale subscriptions against Stripe... 2/10')
+        ->expectsOutput('Total subscriptions checked: 0')
+        ->assertSuccessful();
+});
+
+test('the terminal command passes the fix option to the reconciliation action', function () {
+    SyncStripeSubscriptions::shouldRun()
+        ->once()
+        ->withArgs(fn (bool $fix, ?Closure $onProgress) => $fix === true && $onProgress instanceof Closure)
+        ->andReturn([
+            'total_checked' => 1,
+            'discrepancies' => [[
+                'subscription_id' => 1,
+                'team_id' => 2,
+                'stripe_subscription_id' => 'sub_stale',
+                'stripe_status' => 'canceled',
+                'resolution' => 'manual_review',
+            ]],
+            'resubscribed' => [],
+            'errors' => [],
+            'fixed' => true,
+            'fixed_count' => 0,
+            'manual_review_count' => 1,
+        ]);
+
+    $this->artisan('cloud:sync-stripe-subscriptions', ['--fix' => true])
+        ->expectsOutput('Total subscriptions checked: 1')
+        ->expectsOutput('    Resolution: Manual review required')
+        ->expectsOutput('Automatic corrections applied: 0')
+        ->expectsOutput('Skipped for manual review: 1')
+        ->assertSuccessful();
+});
@@ -1,5 +1,6 @@
 <?php

+use App\Models\Subscription;
 use App\Models\Team;
 use Illuminate\Foundation\Testing\RefreshDatabase;

@@ -14,3 +15,24 @@ test('subscriptionEnded does not throw when team has no subscription', function
    // If we reach here, no exception was thrown
    expect(true)->toBeTrue();
 });
+
+test('subscriptionEnded updates the exact subscription when duplicate rows exist', function () {
+    $team = Team::factory()->create();
+    $otherSubscription = Subscription::create([
+        'team_id' => $team->id,
+        'stripe_subscription_id' => 'sub_other',
+        'stripe_invoice_paid' => true,
+    ]);
+    $subscriptionToEnd = Subscription::create([
+        'team_id' => $team->id,
+        'stripe_subscription_id' => 'sub_stale',
+        'stripe_invoice_paid' => true,
+    ]);
+
+    $team->subscriptionEnded($subscriptionToEnd);
+
+    expect($subscriptionToEnd->fresh()->stripe_subscription_id)->toBeNull()
+        ->and($subscriptionToEnd->fresh()->stripe_invoice_paid)->toBeFalsy()
+        ->and($otherSubscription->fresh()->stripe_subscription_id)->toBe('sub_other')
+        ->and($otherSubscription->fresh()->stripe_invoice_paid)->toBeTruthy();
+});
@@ -301,6 +301,104 @@ describe('POST /api/v1/servers/vultr', function () {
        ]);
    });

+    test('deletes the Vultr instance when local server persistence fails', function () {
+        Http::fake([
+            'https://api.vultr.com/v2/ssh-keys' => Http::response([
+                'ssh_key' => ['id' => 'key-1', 'ssh_key' => testPublicKey()],
+            ], 201),
+            'https://api.vultr.com/v2/ssh-keys*' => Http::response([
+                'ssh_keys' => [],
+                'meta' => ['links' => ['next' => null]],
+            ], 200),
+            'https://api.vultr.com/v2/instances' => Http::response([
+                'instance' => [
+                    'id' => 'instance-persistence-fails',
+                    'main_ip' => '203.0.113.10',
+                    'status' => 'pending',
+                ],
+            ], 202),
+            'https://api.vultr.com/v2/instances/instance-persistence-fails' => Http::response(null, 204),
+        ]);
+
+        $eventDispatcher = Server::getEventDispatcher();
+        Server::setEventDispatcher(clone $eventDispatcher);
+        Server::created(function (): void {
+            throw new RuntimeException('local persistence failed');
+        });
+
+        try {
+            $response = $this->withHeaders([
+                'Authorization' => 'Bearer '.$this->bearerToken,
+                'Content-Type' => 'application/json',
+            ])->postJson('/api/v1/servers/vultr', [
+                'cloud_provider_token_id' => $this->vultrToken->uuid,
+                'region' => 'ewr',
+                'plan' => 'vc2-1c-1gb',
+                'os_id' => 2284,
+                'name' => 'persistence-fails',
+                'private_key_uuid' => $this->privateKey->uuid,
+            ]);
+        } finally {
+            Server::setEventDispatcher($eventDispatcher);
+        }
+
+        $response->assertServerError();
+        $response->assertJson(['message' => 'Failed to create Vultr server.']);
+        $this->assertDatabaseMissing('servers', [
+            'vultr_instance_id' => 'instance-persistence-fails',
+        ]);
+        Http::assertSent(fn ($request) => $request->method() === 'DELETE'
+            && $request->url() === 'https://api.vultr.com/v2/instances/instance-persistence-fails');
+    });
+
+    test('keeps the tracked Vultr server when public IP polling fails', function () {
+        Http::fake([
+            'https://api.vultr.com/v2/ssh-keys' => Http::response([
+                'ssh_key' => ['id' => 'key-1', 'ssh_key' => testPublicKey()],
+            ], 201),
+            'https://api.vultr.com/v2/ssh-keys*' => Http::response([
+                'ssh_keys' => [],
+                'meta' => ['links' => ['next' => null]],
+            ], 200),
+            'https://api.vultr.com/v2/instances' => Http::response([
+                'instance' => [
+                    'id' => 'instance-polling-fails',
+                    'main_ip' => '0.0.0.0',
+                    'status' => 'pending',
+                ],
+            ], 202),
+            'https://api.vultr.com/v2/instances/instance-polling-fails' => Http::response([
+                'error' => 'temporary polling failure',
+            ], 500),
+        ]);
+
+        $response = $this->withHeaders([
+            'Authorization' => 'Bearer '.$this->bearerToken,
+            'Content-Type' => 'application/json',
+        ])->postJson('/api/v1/servers/vultr', [
+            'cloud_provider_token_id' => $this->vultrToken->uuid,
+            'region' => 'ewr',
+            'plan' => 'vc2-1c-1gb',
+            'os_id' => 2284,
+            'name' => 'polling-fails',
+            'private_key_uuid' => $this->privateKey->uuid,
+        ]);
+
+        $response->assertCreated();
+        $response->assertJsonFragment([
+            'vultr_instance_id' => 'instance-polling-fails',
+            'ip' => Server::PLACEHOLDER_IP,
+        ]);
+        $this->assertDatabaseHas('servers', [
+            'name' => 'polling-fails',
+            'ip' => Server::PLACEHOLDER_IP,
+            'team_id' => $this->team->id,
+            'vultr_instance_id' => 'instance-polling-fails',
+            'vultr_instance_status' => 'pending',
+        ]);
+        Http::assertNotSent(fn ($request) => $request->method() === 'DELETE');
+    });
+
    test('server creation authorizes access to the stored Vultr token', function () {
        $member = User::factory()->create();
        $this->team->members()->attach($member->id, ['role' => 'member']);
@@ -8,6 +8,7 @@ use App\Models\Server;
 use App\Models\Team;
 use App\Models\User;
 use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Http\Client\Request;
 use Illuminate\Support\Facades\Http;
 use Livewire\Livewire;

@@ -170,6 +171,56 @@ it('persists the server with a placeholder IP when Vultr has not assigned one ye
        'vultr_instance_id' => 'instance-1',
        'vultr_instance_status' => 'pending',
    ]);
+
+    Http::assertNotSent(fn (Request $request): bool => $request->method() === 'DELETE'
+        && $request->url() === 'https://api.vultr.com/v2/instances/instance-1');
+});
+
+it('deletes the Vultr instance when local server persistence fails', function () {
+    Http::fake([
+        'https://api.vultr.com/v2/ssh-keys' => Http::response([
+            'ssh_key' => ['id' => 'key-1', 'ssh_key' => vultrLivewireTestPublicKey()],
+        ], 201),
+        'https://api.vultr.com/v2/ssh-keys*' => Http::response([
+            'ssh_keys' => [],
+            'meta' => ['links' => ['next' => null]],
+        ], 200),
+        'https://api.vultr.com/v2/instances' => Http::response([
+            'instance' => [
+                'id' => 'instance-1',
+                'label' => 'persistence-fails',
+                'main_ip' => '192.0.2.10',
+                'v6_main_ip' => '2001:db8::1',
+                'status' => 'pending',
+            ],
+        ], 202),
+        'https://api.vultr.com/v2/instances/instance-1' => Http::response(null, 204),
+    ]);
+
+    $eventDispatcher = Server::getEventDispatcher();
+    Server::setEventDispatcher(clone $eventDispatcher);
+    Server::created(function (): void {
+        throw new RuntimeException('local persistence failed');
+    });
+
+    try {
+        Livewire::test(ByVultr::class, ['selectedTokenUuid' => $this->vultrToken->uuid])
+            ->set('server_name', 'persistence-fails')
+            ->set('selected_region', 'ewr')
+            ->set('selected_plan', 'vc2-1c-1gb')
+            ->set('selected_os_id', 2284)
+            ->set('private_key_id', $this->privateKey->id)
+            ->call('submit')
+            ->assertDispatched('error', 'local persistence failed');
+    } finally {
+        Server::setEventDispatcher($eventDispatcher);
+    }
+
+    $this->assertDatabaseMissing('servers', [
+        'vultr_instance_id' => 'instance-1',
+    ]);
+    Http::assertSent(fn (Request $request): bool => $request->method() === 'DELETE'
+        && $request->url() === 'https://api.vultr.com/v2/instances/instance-1');
 });

 it('requires IPv6 when public IPv4 is disabled', function () {
@@ -18,6 +18,20 @@ it('normalizes requested log line counts', function () {
        ->and(normalizeLogLines('50000'))->toBe(10000);
 });

+it('normalizes service resource log line counts before invoking Docker', function (string $controller, mixed $lines, int $expectedLines) {
+    $source = file_get_contents(__DIR__."/../../../app/Http/Controllers/Api/{$controller}.php");
+
+    expect($source)->toContain('$lines = normalizeLogLines($request->query(\'lines\'));')
+        ->and(normalizeLogLines($lines))->toBe($expectedLines);
+})->with([
+    'application invalid value' => ['ServiceApplicationsController', 'invalid', 100],
+    'application negative value' => ['ServiceApplicationsController', '-5', 100],
+    'application value above limit' => ['ServiceApplicationsController', '50000', 10000],
+    'database invalid value' => ['ServiceDatabasesController', 'invalid', 100],
+    'database negative value' => ['ServiceDatabasesController', '-5', 100],
+    'database value above limit' => ['ServiceDatabasesController', '50000', 10000],
+]);
+
 it('parses show_timestamps query values as booleans', function () {
    if (! function_exists('parseLogTimestampFlag')) {
        expect(function_exists('parseLogTimestampFlag'))->toBeTrue();
@@ -0,0 +1,36 @@
+<?php
+
+use App\Models\StandaloneClickhouse;
+use App\Support\ClickhouseBackupCommand;
+
+test('clickhouse exposes scheduled backups', function () {
+    expect((new StandaloneClickhouse)->isBackupSolutionAvailable())->toBeTrue();
+});
+
+test('builds a native clickhouse backup command using container credentials', function () {
+    $commands = ClickhouseBackupCommand::make(
+        containerName: 'clickhouse-uuid',
+        database: 'analytics',
+        archiveName: 'backup-uuid.zip',
+        backupDirectory: '/data/coolify/backups/clickhouse',
+    );
+
+    expect($commands)->toHaveCount(3)
+        ->and($commands[0])->toBe("mkdir -p '/data/coolify/backups/clickhouse'")
+        ->and($commands[1])->toContain("docker exec 'clickhouse-uuid' clickhouse-client")
+        ->and($commands[1])->not->toContain('--password')
+        ->and($commands[1])->toContain("BACKUP DATABASE `analytics` TO File('")
+        ->and($commands[1])->toContain('backup-uuid.zip')
+        ->and($commands[2])->toBe("docker cp 'clickhouse-uuid:/var/lib/clickhouse/backups/backup-uuid.zip' '/data/coolify/backups/clickhouse/backup-uuid.zip'")
+        ->and(ClickhouseBackupCommand::cleanup('clickhouse-uuid', 'backup-uuid.zip'))
+        ->toBe("docker exec 'clickhouse-uuid' rm -f '/var/lib/clickhouse/backups/backup-uuid.zip'");
+});
+
+test('rejects unsafe clickhouse database names', function () {
+    expect(fn () => ClickhouseBackupCommand::make(
+        containerName: 'clickhouse-uuid',
+        database: 'analytics`; DROP DATABASE default; --',
+        archiveName: 'backup-uuid.zip',
+        backupDirectory: '/data/coolify/backups/clickhouse',
+    ))->toThrow(Exception::class);
+});
@@ -1,11 +1,59 @@
 <?php

+use App\Exceptions\RateLimitException;
 use App\Services\DigitalOceanService;
+use Illuminate\Http\Client\RequestException;
 use Illuminate\Support\Facades\Http;
 use Tests\TestCase;

 uses(TestCase::class);

+it('maps final DigitalOcean API error responses', function (int $status, string $message) {
+    Http::fake([
+        'https://api.digitalocean.com/v2/droplets' => Http::response(['message' => $message], $status),
+    ]);
+
+    $exception = null;
+
+    try {
+        (new DigitalOceanService('test-token'))->createDroplet([]);
+    } catch (Throwable $caught) {
+        $exception = $caught;
+    }
+
+    expect($exception)->toBeInstanceOf(Exception::class)
+        ->and($exception)->not->toBeInstanceOf(RequestException::class)
+        ->and($exception->getMessage())->toBe('DigitalOcean API error: '.$message)
+        ->and($exception->getCode())->toBe($status);
+
+    Http::assertSentCount(3);
+})->with([
+    'unauthorized' => [401, 'Unable to authenticate you'],
+    'unprocessable entity' => [422, 'Droplet name is invalid'],
+]);
+
+it('maps a final DigitalOcean rate-limit response with Retry-After', function () {
+    Http::fake([
+        'https://api.digitalocean.com/v2/droplets' => Http::sequence()
+            ->push(['message' => 'Temporary provider error'], 500)
+            ->push(['message' => 'Temporary provider error'], 500)
+            ->push(['message' => 'Too many requests'], 429, ['Retry-After' => '45']),
+    ]);
+
+    try {
+        (new DigitalOceanService('test-token'))->createDroplet([]);
+    } catch (RateLimitException $exception) {
+        expect($exception->getMessage())->toBe('Rate limit exceeded. Please try again later.')
+            ->and($exception->retryAfter)->toBe(45);
+
+        Http::assertSentCount(3);
+
+        return;
+    }
+
+    test()->fail('Expected a RateLimitException to be thrown.');
+});
+
 it('fetches paginated regions from DigitalOcean', function () {
    Http::fake([
        'https://api.digitalocean.com/v2/regions?page=1&per_page=50' => Http::response([
@@ -1,8 +1,12 @@
 <?php

 it('normalizes plus and dotted email identity variants', function () {
-    expect(normalize_email_identity('Ke.VinMcFadden+one@BTInternet.com'))->toBe('kevinmcfadden@btinternet.com');
-    expect(normalize_email_identity('k.e.v.i.n.m.c.f.a.d.d.e.n+two@btinternet.com'))->toBe('kevinmcfadden@btinternet.com');
+    expect(normalize_email_identity('Ke.VinMcFadden+one@Gmail.com'))->toBe('kevinmcfadden@gmail.com');
+    expect(normalize_email_identity('k.e.v.i.n.m.c.f.a.d.d.e.n+two@googlemail.com'))->toBe('kevinmcfadden@googlemail.com');
+});
+
+it('preserves dots and plus suffixes for ordinary email providers', function () {
+    expect(normalize_email_identity(' John.Smith+alerts@Example.com '))->toBe('john.smith+alerts@example.com');
 });

 it('returns null for blank or malformed email identities', function (?string $email) {
@@ -1,6 +1,8 @@
 <?php

+use App\Exceptions\RateLimitException;
 use App\Services\VultrService;
+use Illuminate\Http\Client\RequestException;
 use Illuminate\Support\Facades\Http;
 use Tests\TestCase;

@@ -10,6 +12,52 @@ beforeEach(function () {
    Http::preventStrayRequests();
 });

+it('maps final Vultr API error responses', function (int $status, string $message) {
+    Http::fake([
+        'https://api.vultr.com/v2/instances' => Http::response(['error' => $message], $status),
+    ]);
+
+    $exception = null;
+
+    try {
+        (new VultrService('fake-token'))->createInstance([]);
+    } catch (Throwable $caught) {
+        $exception = $caught;
+    }
+
+    expect($exception)->toBeInstanceOf(Exception::class)
+        ->and($exception)->not->toBeInstanceOf(RequestException::class)
+        ->and($exception->getMessage())->toBe('Vultr API error: '.$message)
+        ->and($exception->getCode())->toBe($status);
+
+    Http::assertSentCount(3);
+})->with([
+    'unauthorized' => [401, 'Unauthorized'],
+    'unprocessable entity' => [422, 'Invalid instance parameters'],
+]);
+
+it('maps a final Vultr rate-limit response with Retry-After', function () {
+    Http::fake([
+        'https://api.vultr.com/v2/instances' => Http::sequence()
+            ->push(['error' => 'Temporary provider error'], 500)
+            ->push(['error' => 'Temporary provider error'], 500)
+            ->push(['error' => 'Too many requests'], 429, ['Retry-After' => '30']),
+    ]);
+
+    try {
+        (new VultrService('fake-token'))->createInstance([]);
+    } catch (RateLimitException $exception) {
+        expect($exception->getMessage())->toBe('Rate limit exceeded. Please try again later.')
+            ->and($exception->retryAfter)->toBe(30);
+
+        Http::assertSentCount(3);
+
+        return;
+    }
+
+    test()->fail('Expected a RateLimitException to be thrown.');
+});
+
 it('gets instances from Vultr API', function () {
    Http::fake([
        'https://api.vultr.com/v2/instances*' => Http::response([