mirror of
https://github.com/coollabsio/coolify.git
synced 2026-11-14 16:52:45 -04:00
Andras Bacsai
d28bc91ff5
Horizon workers kept Server::findCached() results between jobs, so scheduled database backups used stale reachability and subscription state. A backup that was skipped with the stale state counted as done for that day. Flush the Server identity map before each queue job. Add debug logs for the full path of a scheduled run, so a multi-node cloud instance shows where a run is lost: - manager start delay, gap since the previous start, host, pid, memory, and a warning when the start is later than the catch-up window - skip logs with the server state that was used and the current database state - occurrence enqueued, claimed, rejected, completed and failed, with host and the delay since the due time; a warning when the previous run is still open - count of occurrences still open after 15 minutes on each manager run - database backup runs that end without a backup - server connection state changes with the checking node and the SSH error Also keep publishing pending occurrences when one of them fails.
123 lines
4.2 KiB
PHP
123 lines
4.2 KiB
PHP
<?php
|
|
|
|
use App\Jobs\ServerConnectionCheckJob;
|
|
use App\Models\CloudProviderToken;
|
|
use App\Models\PrivateKey;
|
|
use App\Models\Server;
|
|
use App\Models\Team;
|
|
use Illuminate\Foundation\Testing\RefreshDatabase;
|
|
use Illuminate\Support\Facades\Http;
|
|
use Illuminate\Support\Facades\Log;
|
|
use Illuminate\Support\Facades\Process;
|
|
use Illuminate\Support\Facades\Storage;
|
|
|
|
uses(RefreshDatabase::class);
|
|
|
|
function createServerForConnectionIsolationTest(array $attributes = []): Server
|
|
{
|
|
$team = Team::factory()->create();
|
|
$privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);
|
|
|
|
return Server::factory()->create(array_merge([
|
|
'team_id' => $team->id,
|
|
'private_key_id' => $privateKey->id,
|
|
], $attributes));
|
|
}
|
|
|
|
beforeEach(function () {
|
|
Storage::fake('ssh-keys');
|
|
});
|
|
|
|
it('never attempts SSH to a placeholder address', function (string $placeholderIp) {
|
|
$server = createServerForConnectionIsolationTest(['ip' => $placeholderIp]);
|
|
$server->settings->update([
|
|
'is_reachable' => true,
|
|
'is_usable' => true,
|
|
]);
|
|
|
|
Process::fake();
|
|
|
|
(new ServerConnectionCheckJob($server, disableMux: false))->handle();
|
|
|
|
Process::assertNothingRan();
|
|
expect($server->fresh()->unreachable_count)->toBe(0)
|
|
->and((bool) $server->settings->fresh()->is_reachable)->toBeTrue()
|
|
->and((bool) $server->settings->fresh()->is_usable)->toBeTrue();
|
|
})->with([
|
|
'reserved provisioning address' => Server::PLACEHOLDER_IP,
|
|
'unspecified IPv4 address' => '0.0.0.0',
|
|
'unspecified IPv6 address' => '::',
|
|
]);
|
|
|
|
it('does not call cloud provider APIs during an SSH connection check', function () {
|
|
$server = createServerForConnectionIsolationTest(['ip' => '203.0.113.10']);
|
|
$token = CloudProviderToken::create([
|
|
'team_id' => $server->team_id,
|
|
'provider' => 'vultr',
|
|
'token' => 'test-vultr-token',
|
|
'name' => 'Vultr',
|
|
]);
|
|
$server->update([
|
|
'cloud_provider_token_id' => $token->id,
|
|
'vultr_instance_id' => 'instance-1',
|
|
'vultr_instance_status' => 'active',
|
|
]);
|
|
|
|
Http::fake([
|
|
'https://api.vultr.com/*' => Http::response([
|
|
'instance' => ['id' => 'instance-1', 'status' => 'active'],
|
|
]),
|
|
]);
|
|
Process::fake([
|
|
'*' => Process::result(
|
|
output: '{"Server":{"Version":"27.0.0"}}',
|
|
exitCode: 0,
|
|
),
|
|
]);
|
|
|
|
(new ServerConnectionCheckJob($server->fresh(), disableMux: false))->handle();
|
|
|
|
Http::assertNothingSent();
|
|
});
|
|
|
|
it('resets unreachable_count after a successful connection check', function () {
|
|
$server = createServerForConnectionIsolationTest(['ip' => '203.0.113.10']);
|
|
// unreachable_count is not mass-assignable, so it must be set directly.
|
|
$server->unreachable_count = 5;
|
|
$server->save();
|
|
|
|
Process::fake([
|
|
'*' => Process::result(
|
|
output: '{"Server":{"Version":"27.0.0"}}',
|
|
exitCode: 0,
|
|
),
|
|
]);
|
|
|
|
(new ServerConnectionCheckJob($server->fresh(), disableMux: false))->handle();
|
|
|
|
expect($server->fresh()->unreachable_count)->toBe(0);
|
|
});
|
|
|
|
it('logs the checking node and ssh error only when the connection state changes', function () {
|
|
$server = createServerForConnectionIsolationTest(['ip' => '203.0.113.10']);
|
|
$server->settings->update(['is_reachable' => true, 'is_usable' => true]);
|
|
$logPath = tempnam(sys_get_temp_dir(), 'coolify-scheduled-log-');
|
|
config(['logging.channels.scheduled' => ['driver' => 'single', 'path' => $logPath, 'level' => 'debug']]);
|
|
Log::forgetChannel('scheduled');
|
|
Process::fake([
|
|
'*' => Process::result(errorOutput: 'ssh: connect to host 203.0.113.10 port 22: Connection refused', exitCode: 255),
|
|
]);
|
|
|
|
(new ServerConnectionCheckJob($server->fresh(), disableMux: false))->handle();
|
|
(new ServerConnectionCheckJob($server->fresh(), disableMux: false))->handle();
|
|
|
|
$log = file_get_contents($logPath);
|
|
@unlink($logPath);
|
|
|
|
expect(substr_count($log, 'Server connection state changed'))->toBe(1)
|
|
->and($log)->toContain('"is_reachable":false')
|
|
->toContain('"was_reachable":true')
|
|
->toContain('ssh exit 255: ssh: connect to host 203.0.113.10 port 22: Connection refused')
|
|
->toContain('"host":"'.gethostname().'"');
|
|
});
|