Files
ai-app/server/src/files.rs
T

624 lines
24 KiB
Rust

//! Reading and changing files on a configured machine.
//!
//! Every operation here is one small POSIX shell script handed to `Transport`,
//! the way `machines::discover` and `import::list` already ask a machine a
//! question. That is what makes the local and the ssh case one implementation:
//! a second one written against `std::fs` would be the one that gets tested,
//! and the remote half -- the ordering of entries, what a symlink reports, how
//! a permission error reads -- would drift until it shipped broken. The cost is
//! an `sh` process per operation here, which is under a millisecond.
//!
//! The scripts assume GNU coreutils and findutils, which is what
//! `session::import` already assumes. A machine without them fails with that
//! tool's own message, which names what is missing.
//!
//! **The phone names a path, and that is deliberate** -- see PLAN.md's Security
//! section. What is *not* given up: no route here accepts a command. Listing,
//! reading and writing are the fixed scripts below, and the phone chooses only
//! the path and the bytes.
use anyhow::{Context, Result};
use serde::Serialize;
use crate::session::transport::{Input, Launch, Transport};
/// The most of a file that crosses the tunnel, in bytes. Checked on the far
/// machine before anything reads the file, so a 2 GB log costs a `stat` rather
/// than a transfer. A file over it is [`FileRead::TooBig`] with its size,
/// because "we did not read this" and "this is empty" must not look the same.
pub const FILE_LIMIT: u64 = 1024 * 1024;
/// The prelude every script here starts with: the path arrives as `$1`, and
/// this is where a leading `~` becomes that machine's own home.
///
/// The path is a **positional argument** and never text spliced into the script
/// -- the rule `import::find` follows, for the reason `ssh::quote` exists: a
/// path is attacker-adjacent input in a server whose job is running commands,
/// and interpolated it would be syntax rather than data.
///
/// `~` is the one character that costs something for it. A shell expands a
/// tilde in *text*, so a path handed over as an argument arrives with a literal
/// one; expanding it here gives it the same meaning `ssh::quote_path` gives it
/// everywhere else, and it is the *far* machine's `$HOME`. `~user` stays
/// literal and fails with the shell's own message.
///
/// Everything below uses `$p` for the path and `$2` for whatever else.
const PATH_PRELUDE: &str = r#"p=$1; case $p in "~") p=$HOME;; "~/"*) p=$HOME/${p#"~/"};; esac; "#;
/// What a directory turned out to be, and what is in it.
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct Listing {
/// `pwd -P` of the directory that was listed. Answered by the machine
/// rather than worked out here, so the phone navigates on a resolved
/// absolute path: the parent of one of these is a string operation, and a
/// `~` a session was spawned with is shown as what it turned out to be.
pub path: String,
pub entries: Vec<Entry>,
}
#[derive(Debug, Serialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub struct Entry {
pub name: String,
/// What tapping it does, which for a symlink is decided by its *target* --
/// a link to a directory navigates.
pub kind: EntryKind,
pub size: u64,
pub modified: i64,
/// Whether the entry itself is a symlink, whatever [`Entry::kind`] says
/// its target is.
pub link: bool,
}
#[derive(Debug, Serialize, PartialEq, Eq, Clone, Copy)]
#[serde(rename_all = "camelCase")]
pub enum EntryKind {
Directory,
File,
/// A socket, a device, a fifo -- and a symlink whose target is missing or
/// loops, which `find` reports the same way. Shown, because a directory
/// that hid what it held would be lying about being empty.
Other,
}
/// What reading a file produced -- four answers, not content-or-error.
///
/// A binary file drawn as text and a big file cut off silently are both wrong
/// in ways the reader cannot see, and "couldn't read it" must not look like
/// "it is empty". A genuinely empty file is [`FileRead::Text`] with nothing in
/// it, which is what it is.
#[derive(Debug, Serialize)]
#[serde(tag = "kind", rename_all = "camelCase")]
pub enum FileRead {
Text {
size: u64,
modified: i64,
/// What [`write`] is given back to prove the file has not moved on.
sha256: String,
content: String,
},
/// Not UTF-8. Its size is reported; nothing is shown.
Binary { size: u64, modified: i64 },
/// Over [`FILE_LIMIT`]. Its size is reported, so the reader knows what they
/// are looking at rather than only that they cannot have it.
TooBig { size: u64, modified: i64 },
}
/// What a file is after being written, so the editor's precondition is
/// fresh without a second read.
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct Written {
pub size: u64,
pub modified: i64,
pub sha256: String,
}
/// The exit code the write script uses for "this is not the file you read",
/// which the route turns into a 409. Distinct from every other failure, which
/// is a message from the machine.
pub const STALE: i32 = 3;
/// A path the phone may name: absolute, or home-relative on that machine.
///
/// Shared with `POST /sessions/{id}/cwd`, which asks the same question for the
/// same reason -- a relative path is relative to something nobody looking at
/// the screen can see, so it is refused rather than resolved against a guess.
pub fn check_path(path: &str) -> Result<String> {
let path = path.trim();
if path.is_empty() {
anyhow::bail!("that is a path with nothing in it");
}
if !path.starts_with('/') && !path.starts_with('~') {
anyhow::bail!(
"{path} is not an absolute path, so where it would be depends on something \
nobody looking at this can see -- start it with / or ~"
);
}
Ok(path.to_string())
}
/// Runs one of the scripts below with `path` as `$1` and `extra` as `$2`.
fn launch(script: String, path: &str, extra: Option<&str>) -> Launch {
let mut args = vec![
"-c".to_string(),
script,
// `$0`, which is what `sh` names itself in a message about the script;
// the path is `$1`.
"sh".to_string(),
path.to_string(),
];
args.extend(extra.map(str::to_string));
Launch::new("sh", args, None)
}
/// Everything in `path`, and what `path` resolved to. Entries are separated by
/// `\0` and their fields by `\t`, so a filename with a newline or a tab in it
/// survives -- both are legal, and a listing that lost one would quietly show
/// the wrong thing.
pub async fn list(transport: &Transport, path: &str) -> Result<Listing> {
let script = format!(
"{PATH_PRELUDE}cd -- \"$p\" && pwd -P && \
find . -mindepth 1 -maxdepth 1 -printf '%y\\t%Y\\t%s\\t%T@\\t%f\\0'"
);
let out = transport
.capture_with_input(&launch(script, path, None), Input::None)
.await?
.ok()?;
let out = String::from_utf8_lossy(&out);
let (resolved, entries) = out
.split_once('\n')
.context("the machine did not say which directory it listed")?;
Ok(Listing {
path: resolved.to_string(),
entries: parse_entries(entries),
})
}
/// The `find` output above, as rows. A record without all five fields is
/// dropped rather than guessed at: it can only come from a `find` that printed
/// something else, and half a row is worse than no row.
fn parse_entries(text: &str) -> Vec<Entry> {
text.split('\0')
.filter(|record| !record.is_empty())
.filter_map(|record| {
// Five, so that a name containing a tab keeps it: `%f` is last
// exactly so the split can stop.
let mut fields = record.splitn(5, '\t');
let own = fields.next()?;
let target = fields.next()?;
let size = fields.next()?.parse().ok()?;
// `%T@` is seconds with a fractional part; the phone shows a date.
let modified = fields.next()?.split('.').next()?.parse().ok()?;
let name = fields.next()?;
Some(Entry {
name: name.to_string(),
kind: match target {
"d" => EntryKind::Directory,
"f" => EntryKind::File,
_ => EntryKind::Other,
},
size,
modified,
link: own == "l",
})
})
.collect()
}
/// One file's content, or the reason there is none to show.
///
/// The size is checked on the far machine *before* anything reads the file, so
/// a file over [`FILE_LIMIT`] costs a `stat` rather than a transfer. `stat -L`
/// and `sha256sum` both follow symlinks, as `cat` does.
pub async fn read(transport: &Transport, path: &str) -> Result<FileRead> {
// Two header lines, then the bytes: `<size> <mtime>`, then either `tooBig`
// or the digest. A header rather than a JSON envelope because the content
// is bytes and may not be text at all.
let script = format!(
"{PATH_PRELUDE}set -e; \
h=$(stat -L -c '%s %Y' -- \"$p\"); \
case ${{h%% *}} in *[!0-9]*) exit 1;; esac; \
if [ \"${{h%% *}}\" -gt {FILE_LIMIT} ]; then printf '%s\\ntooBig\\n' \"$h\"; exit 0; fi; \
d=$(sha256sum -- \"$p\"); \
printf '%s\\n%s\\n' \"$h\" \"${{d%% *}}\"; \
cat -- \"$p\""
);
let out = transport
.capture_with_input(&launch(script, path, None), Input::None)
.await?
.ok()?;
let (size, modified, second, content) = split_read(&out)?;
if second == "tooBig" {
return Ok(FileRead::TooBig { size, modified });
}
Ok(match String::from_utf8(content.to_vec()) {
Ok(content) => FileRead::Text {
size,
modified,
sha256: second.to_string(),
content,
},
Err(_) => FileRead::Binary { size, modified },
})
}
/// The read script's two header lines and the bytes after them.
fn split_read(out: &[u8]) -> Result<(u64, i64, &str, &[u8])> {
let missing = || anyhow::anyhow!("the machine did not describe the file it read");
let first = out.iter().position(|b| *b == b'\n').ok_or_else(missing)?;
let rest = &out[first + 1..];
let second = rest.iter().position(|b| *b == b'\n').ok_or_else(missing)?;
let header = std::str::from_utf8(&out[..first]).map_err(|_| missing())?;
let (size, modified) = header.split_once(' ').ok_or_else(missing)?;
Ok((
size.trim().parse()?,
modified.trim().parse()?,
std::str::from_utf8(&rest[..second]).map_err(|_| missing())?,
&rest[second + 1..],
))
}
/// Replaces `path`'s contents, but only while it still hashes to `expected`.
///
/// Agents edit files while people read them, so a stale copy landing on top of
/// somebody else's edit is the common case rather than the exotic one. The
/// digest the reader was shown is compared on the machine, and a file that has
/// moved on comes back as [`STALE`] rather than being overwritten.
///
/// A temp file and a rename, so a connection dropped mid-write leaves the old
/// file whole, and `chmod --reference` so the mode survives -- an executable
/// script written as a fresh file would stop being one. What that trades away:
/// the inode changes, so a hard link elsewhere stops being the same file.
///
/// The check and the write are **not** atomic against a writer landing between
/// them -- a window of microseconds on that machine. Accepted: the alternative
/// is a lock this has no way to make every other writer take.
pub async fn write(
transport: &Transport,
path: &str,
expected: &str,
bytes: Vec<u8>,
) -> Result<Result<Written, Stale>> {
let script = format!(
"{PATH_PRELUDE}set -e; \
have=$(sha256sum -- \"$p\"); \
[ \"${{have%% *}}\" = \"$2\" ] || exit {STALE}; \
cat > \"$p.ai-app-tmp\"; \
chmod --reference=\"$p\" \"$p.ai-app-tmp\"; \
mv -f -- \"$p.ai-app-tmp\" \"$p\"; \
stat -L -c '%s %Y' -- \"$p\"; \
now=$(sha256sum -- \"$p\"); \
printf '%s\\n' \"${{now%% *}}\""
);
let captured = transport
.capture_with_input(&launch(script, path, Some(expected)), Input::Bytes(bytes))
.await?;
if captured.status.code() == Some(STALE) {
return Ok(Err(Stale));
}
let out = String::from_utf8_lossy(&captured.ok()?).into_owned();
let mut lines = out.lines();
let missing = || anyhow::anyhow!("the machine did not describe the file it wrote");
let header = lines.next().ok_or_else(missing)?;
let (size, modified) = header.split_once(' ').ok_or_else(missing)?;
Ok(Ok(Written {
size: size.trim().parse()?,
modified: modified.trim().parse()?,
sha256: lines.next().ok_or_else(missing)?.trim().to_string(),
}))
}
/// The file is not the one that was read. Its own type rather than an error
/// string, because the route answers it with a different status and the phone
/// with a different question.
#[derive(Debug)]
pub struct Stale;
/// Creates an empty file, refusing to truncate one that is already there.
/// `set -C` is the shell's own noclobber, so an existing name fails with the
/// shell's own message rather than with a check that could race the redirection
/// it is guarding.
pub async fn create_file(transport: &Transport, path: &str) -> Result<()> {
let script = format!("{PATH_PRELUDE}set -C; : > \"$p\"");
transport
.capture_with_input(&launch(script, path, None), Input::None)
.await?
.ok()?;
Ok(())
}
/// Creates a directory. Plain `mkdir`, not `-p`, for the reason
/// [`create_file`] sets noclobber: a name that exists is something the person
/// typing it should be told about.
pub async fn create_dir(transport: &Transport, path: &str) -> Result<()> {
let script = format!("{PATH_PRELUDE}mkdir -- \"$p\"");
transport
.capture_with_input(&launch(script, path, None), Input::None)
.await?
.ok()?;
Ok(())
}
/// The stdout of a script that must have succeeded, as text.
#[cfg(test)]
fn text(captured: crate::session::transport::Captured) -> Result<String> {
Ok(String::from_utf8_lossy(&captured.ok()?).into_owned())
}
#[cfg(test)]
mod tests {
use super::*;
/// The names a listing has to survive. All four are legal, and each one
/// broke a listing somewhere before it was separated with `\0`.
#[test]
fn a_listing_survives_the_names_a_filesystem_allows() {
let record = |own: &str, target: &str, size: &str, time: &str, name: &str| {
format!("{own}\t{target}\t{size}\t{time}\t{name}\0")
};
let text = [
record("f", "f", "12", "1756900000.5", "with\ta tab"),
record("f", "f", "13", "1756900001.0", "with\na newline"),
record("d", "d", "4096", "1756900002.0", "-leading-dash"),
record("l", "d", "7", "1756900003.9", "it's a link"),
record("l", "N", "7", "1756900004.0", "dangling"),
]
.concat();
let entries = parse_entries(&text);
assert_eq!(
entries.iter().map(|e| e.name.as_str()).collect::<Vec<_>>(),
[
"with\ta tab",
"with\na newline",
"-leading-dash",
"it's a link",
"dangling",
]
);
assert_eq!(entries[0].kind, EntryKind::File);
assert_eq!(entries[0].modified, 1756900000);
assert_eq!(entries[2].kind, EntryKind::Directory);
assert_eq!(entries[2].size, 4096);
// The kind is the target's, so a link to a directory navigates -- and
// one whose target is gone is neither a file nor a directory.
assert!(entries[3].link);
assert_eq!(entries[3].kind, EntryKind::Directory);
assert_eq!(entries[4].kind, EntryKind::Other);
}
#[test]
fn a_half_written_record_is_dropped_rather_than_guessed_at() {
assert!(parse_entries("f\tf\t12\0").is_empty());
assert!(parse_entries("").is_empty());
assert!(parse_entries("\0\0").is_empty());
}
#[test]
fn a_path_is_absolute_or_home_relative() {
assert_eq!(check_path(" /etc/hosts ").unwrap(), "/etc/hosts");
assert_eq!(check_path("~/repos").unwrap(), "~/repos");
assert!(check_path("").is_err());
assert!(check_path(" ").is_err());
let refused = check_path("repos/ai-app").unwrap_err().to_string();
assert!(refused.contains("repos/ai-app"), "{refused}");
assert!(refused.contains("start it with / or ~"), "{refused}");
}
/// The scripts, against a real tree, through the transport that runs them
/// here -- cheap, because `sh` is wherever `cargo test` is. The remote
/// transport runs the identical text.
fn tree() -> tempfile::TempDir {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("hello.txt"), "one\ntwo\n").unwrap();
std::fs::write(dir.path().join("it's a file"), "quoted\n").unwrap();
std::fs::write(dir.path().join("binary.bin"), [0xff, 0xfe, 0x00, 0x01]).unwrap();
std::fs::create_dir(dir.path().join("sub")).unwrap();
dir
}
fn at(dir: &tempfile::TempDir, name: &str) -> String {
dir.path().join(name).to_string_lossy().into_owned()
}
#[tokio::test]
async fn a_directory_lists_with_its_resolved_path() {
let dir = tree();
let listing = list(&Transport::Here, &dir.path().to_string_lossy())
.await
.unwrap();
// `pwd -P`, so a temp directory reached through a symlinked /tmp
// answers with what it really is.
assert!(listing.path.starts_with('/'), "{}", listing.path);
let mut names: Vec<&str> = listing.entries.iter().map(|e| e.name.as_str()).collect();
names.sort_unstable();
assert_eq!(names, ["binary.bin", "hello.txt", "it's a file", "sub"]);
}
#[tokio::test]
async fn a_missing_directory_fails_with_the_machine_s_own_message() {
let dir = tree();
let err = list(&Transport::Here, &at(&dir, "nope"))
.await
.unwrap_err()
.to_string();
assert!(err.contains("nope"), "{err}");
}
#[tokio::test]
async fn text_binary_and_too_big_are_three_different_answers() {
let dir = tree();
let read_at = async |name: &str| read(&Transport::Here, &at(&dir, name)).await.unwrap();
match read_at("hello.txt").await {
FileRead::Text {
size,
content,
sha256,
..
} => {
assert_eq!(size, 8);
assert_eq!(content, "one\ntwo\n");
assert_eq!(sha256.len(), 64);
}
other => panic!("{other:?}"),
}
assert!(matches!(
read_at("binary.bin").await,
FileRead::Binary { size: 4, .. }
));
std::fs::write(dir.path().join("big"), vec![b'x'; FILE_LIMIT as usize + 1]).unwrap();
assert!(matches!(read_at("big").await, FileRead::TooBig { .. }));
// Empty is text with nothing in it -- not a fourth state, and not the
// same as any of the three above.
std::fs::write(dir.path().join("empty"), "").unwrap();
assert!(matches!(
read_at("empty").await,
FileRead::Text { size: 0, .. }
));
}
#[tokio::test]
async fn a_write_lands_only_while_the_file_is_the_one_that_was_read() {
let dir = tree();
let path = at(&dir, "hello.txt");
let FileRead::Text { sha256, .. } = read(&Transport::Here, &path).await.unwrap() else {
panic!("expected text");
};
let written = write(&Transport::Here, &path, &sha256, b"three\n".to_vec())
.await
.unwrap()
.expect("nothing had changed it");
assert_eq!(written.size, 6);
assert_eq!(std::fs::read_to_string(&path).unwrap(), "three\n");
// The same digest again, against a file that has moved on: the
// agent-edits-while-you-read case, which must refuse rather than
// overwrite.
std::fs::write(&path, "somebody else\n").unwrap();
assert!(
write(&Transport::Here, &path, &sha256, b"mine\n".to_vec())
.await
.unwrap()
.is_err()
);
assert_eq!(std::fs::read_to_string(&path).unwrap(), "somebody else\n");
}
#[tokio::test]
async fn a_write_keeps_the_mode_it_found() {
use std::os::unix::fs::PermissionsExt;
let dir = tree();
let path = at(&dir, "script.sh");
std::fs::write(&path, "#!/bin/sh\n").unwrap();
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap();
let FileRead::Text { sha256, .. } = read(&Transport::Here, &path).await.unwrap() else {
panic!("expected text");
};
write(
&Transport::Here,
&path,
&sha256,
b"#!/bin/sh\ntrue\n".to_vec(),
)
.await
.unwrap()
.unwrap();
let mode = std::fs::metadata(&path).unwrap().permissions().mode() & 0o777;
assert_eq!(mode, 0o755, "an executable script stopped being one");
}
#[tokio::test]
async fn creating_refuses_a_name_that_is_already_there() {
let dir = tree();
create_file(&Transport::Here, &at(&dir, "new.txt"))
.await
.unwrap();
assert_eq!(std::fs::read_to_string(at(&dir, "new.txt")).unwrap(), "");
assert!(
create_file(&Transport::Here, &at(&dir, "new.txt"))
.await
.is_err()
);
// And the file that was there is untouched.
std::fs::write(at(&dir, "new.txt"), "mine\n").unwrap();
assert!(
create_file(&Transport::Here, &at(&dir, "new.txt"))
.await
.is_err()
);
assert_eq!(
std::fs::read_to_string(at(&dir, "new.txt")).unwrap(),
"mine\n"
);
create_dir(&Transport::Here, &at(&dir, "made"))
.await
.unwrap();
assert!(at(&dir, "made").parse::<std::path::PathBuf>().is_ok());
assert!(
create_dir(&Transport::Here, &at(&dir, "made"))
.await
.is_err()
);
}
/// A path that tries to close the quote and start a command of its own. It
/// is an argument rather than syntax, so it stays one absurd filename -- the
/// same property `ssh.rs` tests for the remote side.
#[tokio::test]
async fn a_path_full_of_shell_crosses_as_data() {
let dir = tree();
let evil = at(&dir, "'; touch pwned; '");
create_file(&Transport::Here, &evil).await.unwrap();
assert!(std::path::Path::new(&evil).is_file());
assert!(!dir.path().join("pwned").exists());
let listing = list(&Transport::Here, &dir.path().to_string_lossy())
.await
.unwrap();
assert!(
listing
.entries
.iter()
.any(|e| e.name == "'; touch pwned; '")
);
}
/// The tilde is the one character the prelude gives a meaning, and it is the
/// *machine's* home -- here, this one.
#[tokio::test]
async fn a_leading_tilde_means_the_machine_s_own_home() {
let Some(home) = std::env::home_dir() else {
return;
};
let script = format!("{PATH_PRELUDE}printf '%s' \"$p\"");
let captured = Transport::Here
.capture_with_input(&launch(script.clone(), "~/repos", None), Input::None)
.await
.unwrap();
assert_eq!(
text(captured).unwrap(),
home.join("repos").to_string_lossy()
);
let captured = Transport::Here
.capture_with_input(&launch(script.clone(), "~", None), Input::None)
.await
.unwrap();
assert_eq!(text(captured).unwrap(), home.to_string_lossy());
// Leading, and its own segment only -- `ssh::quote_path`'s rule.
for literal in ["/tmp/~/x", "~user/x"] {
let captured = Transport::Here
.capture_with_input(&launch(script.clone(), literal, None), Input::None)
.await
.unwrap();
assert_eq!(text(captured).unwrap(), literal);
}
}
}