coolify/tests/Feature/RemoteProcessTimeoutTest.php

<?php

use App\Helpers\SshMultiplexingHelper;
use App\Models\PrivateKey;
use App\Models\Server;
use App\Models\User;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\Process;
use Illuminate\Support\Facades\Storage;

/**
 * Tests that per-call SSH command timeouts (e.g. a scheduled backup's configured
 * timeout) reach the shell-level `timeout N ssh` wrapper instead of being capped
 * by the global constants.ssh.command_timeout default.
 *
 * @see https://github.com/coollabsio/coolify DatabaseBackupJob/VolumeBackupJob pass
 *      a per-backup timeout to instant_remote_process()
 */
uses(RefreshDatabase::class);

function makeTimeoutTestServer(): Server
{
    $user = User::factory()->create();
    $team = $user->teams()->first();

    $privateKeyContent = '-----BEGIN OPENSSH PRIVATE KEY-----
'.
        'b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
'.
        'QyNTUxOQAAACBbhpqHhqv6aI67Mj9abM3DVbmcfYhZAhC7ca4d9UCevAAAAJi/QySHv0Mk
'.
        'hwAAAAtzc2gtZWQyNTUxOQAAACBbhpqHhqv6aI67Mj9abM3DVbmcfYhZAhC7ca4d9UCevA
'.
        'AAAECBQw4jg1WRT2IGHMncCiZhURCts2s24HoDS0thHnnRKVuGmoeGq/pojrsyP1pszcNV
'.
        'uZx9iFkCELtxrh31QJ68AAAAEXNhaWxANzZmZjY2ZDJlMmRkAQIDBA==
'.
        '-----END OPENSSH PRIVATE KEY-----';

    $privateKey = PrivateKey::create([
        'name' => 'timeout-test-key-'.uniqid(),
        'private_key' => $privateKeyContent,
        'team_id' => $team->id,
    ]);

    Storage::fake('ssh-keys');
    Storage::disk('ssh-keys')->put("ssh_key@{$privateKey->uuid}", $privateKeyContent);

    $server = Server::factory()->create([
        'team_id' => $team->id,
        'private_key_id' => $privateKey->id,
    ]);
    // These tests count ssh processes; the host key check would add a key scan.
    $server->settings->forceFill(['is_ssh_host_key_check_disabled' => true])->save();

    Storage::disk('ssh-keys')->put("ssh_key@{$server->privateKey->uuid}", $server->privateKey->private_key);

    return $server;
}

it('wraps ssh commands with an explicitly passed command timeout', function () {
    config(['constants.ssh.mux_enabled' => false]);
    $server = makeTimeoutTestServer();

    $command = SshMultiplexingHelper::generateSshCommand($server, 'echo ok', commandTimeout: 7200);

    expect($command)->toStartWith('timeout 7200 ssh ');
});

it('wraps ssh commands with the configured default timeout when none is passed', function () {
    config([
        'constants.ssh.mux_enabled' => false,
        'constants.ssh.command_timeout' => 1234,
    ]);
    $server = makeTimeoutTestServer();

    $command = SshMultiplexingHelper::generateSshCommand($server, 'echo ok');

    expect($command)->toStartWith('timeout 1234 ssh ');
});

it('forwards the per-call timeout of instant_remote_process to the ssh timeout wrapper', function () {
    config(['constants.ssh.mux_enabled' => false]);
    $server = makeTimeoutTestServer();

    Process::fake();

    instant_remote_process(['echo ok'], $server, timeout: 7200, disableMultiplexing: true);

    Process::assertRan(fn ($process) => str_starts_with($process->command, 'timeout 7200 ssh '));
});

it('uses the configured default timeout in instant_remote_process when no timeout is passed', function () {
    config([
        'constants.ssh.mux_enabled' => false,
        'constants.ssh.command_timeout' => 1234,
    ]);
    $server = makeTimeoutTestServer();

    Process::fake();

    instant_remote_process(['echo ok'], $server, disableMultiplexing: true);

    Process::assertRan(fn ($process) => str_starts_with($process->command, 'timeout 1234 ssh '));
});

it('does not replay a command batch when a remote command fails with connection text in its output', function () {
    config([
        'constants.ssh.mux_enabled' => false,
        'constants.ssh.max_retries' => 3,
        'constants.ssh.retry_base_delay' => 0,
    ]);
    $server = makeTimeoutTestServer();

    Process::fake([
        '*' => Process::result(errorOutput: 'mc: <ERROR> Unable to initialize new alias. dial tcp 10.0.0.5:9000: connect: connection refused', exitCode: 1),
    ]);

    expect(fn () => instant_remote_process(['docker run -d --name backup-of-x minio/mc', 'docker exec backup-of-x mc alias set temporary http://s3 k s'], $server, disableMultiplexing: true))
        ->toThrow(RuntimeException::class, 'connection refused');

    Process::assertRanTimes(fn ($process) => str_contains($process->command, 'ssh '), 1);
});

it('retries a command batch when the ssh connection itself fails', function () {
    config([
        'constants.ssh.mux_enabled' => false,
        'constants.ssh.max_retries' => 3,
        'constants.ssh.retry_base_delay' => 0,
    ]);
    $server = makeTimeoutTestServer();

    Process::fake([
        '*' => Process::sequence()
            ->push(Process::result(errorOutput: 'kex_exchange_identification: read: Connection reset by peer', exitCode: 255))
            ->push(Process::result(output: 'ok')),
    ]);

    expect(instant_remote_process(['echo ok'], $server, disableMultiplexing: true))->toBe('ok');

    Process::assertRanTimes(fn ($process) => str_contains($process->command, 'ssh '), 2);
});