diff --git a/server/src/session/transport.rs b/server/src/session/transport.rs index b075c75..99ca972 100644 --- a/server/src/session/transport.rs +++ b/server/src/session/transport.rs @@ -21,12 +21,31 @@ use std::path::{Path, PathBuf}; use std::process::Stdio; +use std::sync::LazyLock; use anyhow::{Context, Result}; use tokio::process::Child; +use tokio::sync::Semaphore; use crate::config::SshConfig; +/// How many [`Transport::capture`] calls may have an ssh connection open at +/// once, across every setup. +/// +/// `capture` is what a batch (deleting several imports, listing several +/// machines) fans out over -- one child `ssh` process per call, all started +/// within the same tick. Nothing here throttled that, so a batch large +/// enough to open more connections than the remote sshd's default +/// `MaxStartups` (10, before it starts randomly refusing) has some of +/// them come back as "Connection closed" -- not a real failure of the +/// operation, just too many handshakes landing on the listener at once. Four +/// keeps a batch comfortably under that ceiling while still overlapping the +/// network round trips. Long-lived processes (`Transport::spawn`, a +/// session's own child) don't take a permit: they hold it for the session's +/// lifetime rather than for one round trip, which would starve every other +/// probe behind it. +pub(crate) static CAPTURE_PERMITS: LazyLock = LazyLock::new(|| Semaphore::new(4)); + /// What a driver needs run in order to exist as a process. /// /// Deliberately just the three things every transport can carry. Anything diff --git a/server/src/setups.rs b/server/src/setups.rs index b3d3a82..61d3543 100644 --- a/server/src/setups.rs +++ b/server/src/setups.rs @@ -163,6 +163,13 @@ pub fn tidy(value: &str) -> Option { /// Runs a launch to completion and returns its stdout. impl Transport { pub async fn capture(&self, launch: &Launch) -> Result { + // See `CAPTURE_PERMITS`: caps how many of these run their ssh + // connection at once, so a batch doesn't open more than the remote + // sshd tolerates before it starts dropping them. + let _permit = super::session::transport::CAPTURE_PERMITS + .acquire() + .await + .expect("capture semaphore is never closed"); let child = self.spawn(launch, super::session::transport::Streams::Piped)?; let output = child .wait_with_output()