getCode(); if (is_int($exitCode) && $exitCode > 0 && $exitCode !== 255) { return false; } return $this->isRetryableSshError($exception->getMessage()); } /** * Calculate delay for exponential backoff */ protected function calculateRetryDelay(int $attempt): int { $baseDelay = config('constants.ssh.retry_base_delay'); $maxDelay = config('constants.ssh.retry_max_delay'); $multiplier = config('constants.ssh.retry_multiplier'); $delay = min($baseDelay * pow($multiplier, $attempt), $maxDelay); return (int) $delay; } /** * Execute a callback with SSH retry logic * * @param callable $callback The operation to execute * @param array $context Context for logging (server, command, etc.) * @param bool $throwError Whether to throw error on final failure * @return mixed The result from the callback */ protected function executeWithSshRetry(callable $callback, array $context = [], bool $throwError = true) { $maxRetries = config('constants.ssh.max_retries'); $lastError = null; $lastErrorMessage = ''; // Randomly fail the command with a key exchange error for testing // if (random_int(1, 10) === 1) { // 10% chance to fail // throw new \RuntimeException('SSH key exchange failed: kex_exchange_identification: read: Connection reset by peer'); // } for ($attempt = 0; $attempt < $maxRetries; $attempt++) { try { return $callback(); } catch (\Throwable $e) { $lastError = $e; $lastErrorMessage = $e->getMessage(); // Check if it's retryable and not the last attempt if ($this->isRetryableSshFailure($e) && $attempt < $maxRetries - 1) { $delay = $this->calculateRetryDelay($attempt); // Add deployment log if available (for ExecuteRemoteCommand trait) if (isset($this->application_deployment_queue) && method_exists($this, 'addRetryLogEntry')) { $this->addRetryLogEntry($attempt + 1, $maxRetries, $delay, $lastErrorMessage); } sleep($delay); continue; } // Not retryable or max retries reached break; } } // All retries exhausted if ($attempt >= $maxRetries) { Log::error('SSH operation failed after all retries', array_merge($context, [ 'attempts' => $attempt, 'error' => $lastErrorMessage, ])); } if ($throwError && $lastError) { // If the error message is empty, provide a more meaningful one if (empty($lastErrorMessage) || trim($lastErrorMessage) === '') { $contextInfo = isset($context['server']) ? " to server {$context['server']}" : ''; $attemptInfo = $attempt > 1 ? " after {$attempt} attempts" : ''; throw new \RuntimeException("SSH connection failed{$contextInfo}{$attemptInfo}", $lastError->getCode()); } throw $lastError; } return null; } }