//! Runs an app's optional on-demand build step and tracks it, so the phone //! can be shown "building..." rather than silently downloading a stale APK. //! //! Nothing here knows what any particular build does: it is a command from //! the config plus a two-file staleness rule (see //! `crate::config::Component`). The build runs in the background and //! the routes answer immediately either way, so a multi-minute build never //! holds a request open past a phone's read timeout. use std::collections::{HashMap, VecDeque}; use std::path::PathBuf; use std::process::Command; use std::sync::{Arc, Mutex}; use serde::Serialize; use crate::config::Component; /// What a build has got through, in whatever it counts in. #[derive(Clone, Copy, Serialize)] #[serde(rename_all = "camelCase")] pub struct Progress { pub done: u64, pub total: u64, } /// The marker a build prints to report progress, at the start of a line. /// /// Recognised on every build's output rather than switched on by a config /// key: a build opts in by printing it, and one more `starts_with` per line /// is not worth a field, an acceptance, and a manifest flag to avoid. /// Nothing here knows what the numbers count -- Gradle tasks, files, /// anything -- only that the second is the whole of it. const PROGRESS_MARKER: &str = "@@progress "; /// What is known about whether a component's build is current. /// /// Three states rather than a boolean, because "we cannot tell" must not /// share a value with either answer. A project this server has never /// built, one outside git, and one with uncommitted work in its directory /// are all *unknown* -- and an unknown drawn as `Current` would be the /// exact failure this was added to stop. #[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize)] #[serde(rename_all = "camelCase")] pub enum Freshness { /// Built from the commit its own directory is on. Current, /// Its directory has moved past the commit it was built from. Behind, /// Never built here, no checkout to compare against, or uncommitted /// work in its directory. Unknown, } /// How a finished component reports the commit it was built from. /// /// A callback rather than a handle to the app list: `BuildState` lives /// *inside* that list, so holding it would be a cycle, and the single fact /// a build needs to report upward does not justify one. The route that /// starts a build has the list and supplies this. pub type RecordBuilt = Arc; /// The handles a build shares with the rest of the server, as opposed to /// the project configuration it is built from. /// /// Grouped because they travel together and because they are a different /// kind of thing from the rest of the arguments: everything else describes /// *this project*, while these two are the server's own state that a build /// happens to touch -- a pull marking a checkout current, and an exec /// waiting for a download. #[derive(Clone)] pub struct Shared { pub remote_checks: crate::git::RemoteChecks, pub downloads: Arc, } #[derive(Default)] struct Inner { /// What each component was last built from, by component name. /// /// Seeded from the config at construction and updated as a run /// records. Kept here rather than read back off `components` because /// this state is deliberately *carried across* a config write -- see /// [`BuildState::matches`] -- so its own copy of the components goes /// stale the moment a build records against it. built_from: HashMap, /// A pull is running. The one thing here that is the *project's* and /// not a component's: there is one checkout, and a pull rewrites the /// files every component builds from, so it is exclusive with all of /// them. /// /// Components have no such flag between them -- whether one is being /// built is a property of its own entry in `runs`, so building one /// neither blocks nor says anything about the others. pulling: bool, /// What the *project* is doing: fetching, pulling. Work belonging to /// one component is in `runs` instead, because that is where the card /// shows it -- a progress bar under the whole project could only ever /// say that something, somewhere, was happening. phase: Option, /// One entry per component that has been built since this server /// started, in the order they were first reached. /// /// Kept after a run rather than cleared at the start of the next one, /// because a run is now one component's: clearing the list would /// throw away a sibling's outcome to report on something that has /// nothing to do with it. An entry is replaced only when *that* /// component is claimed again. runs: Vec, /// The failure from the last pull, cleared when any new work starts. /// Kept rather than logged-and-dropped because the phone is where /// this is being driven from and usually has no access to the /// server's log. /// /// Pulls only. A build failure belongs to the component that produced /// it (`ComponentRun::error`) -- held here it was the project's one /// error slot, so two components building at once had one place to /// report two outcomes. error: Option, /// Whether that failure was a pull with no fast-forward to make, /// which is the one the phone can offer a way past. Beside the /// message rather than inside it because the phone acts on it: a /// button that appears only when git happened to phrase itself a /// certain way is a button nobody can rely on. unrelated_histories: bool, } impl Inner { /// Whether this component is being worked on right now. /// /// Its own entry having an open step, which is the same thing the /// phone reads off `ComponentStatus::step` -- so what disables a /// component's button here and what draws its bar there cannot come /// to disagree. fn component_running(&self, name: &str) -> bool { self.runs .iter() .any(|run| run.name == name && run.step.is_some()) } /// Whether anything at all is happening for this project. /// /// What `building` means on the wire, and deliberately a derived /// answer rather than a flag of its own: a flag would be a second /// place for the truth, and the one it disagreed with would be the /// per-component one everything else is now drawn from. fn anything_running(&self) -> bool { self.pulling || self.runs.iter().any(|run| run.step.is_some()) } } /// What one component's part of the build is doing, or did. /// /// Per component rather than one set of numbers for the whole project, /// because that is how it is read: a progress bar and the lines scrolling /// under it belong to the thing being built, and shown under the project /// they could only say that *something* was happening. It is also what /// lets two components build at once without their output interleaving /// into one stream. struct ComponentRun { name: String, /// What it is doing now -- building, installing, restarting -- or /// `None` once it has finished. step: Option, /// Steps done and steps total, when the running command reports them. progress: Option<(u64, u64)>, /// The tail of this component's output. Bounded because this is a /// progress display, not a log file -- `crate::logs` has the whole of /// it, per component, on disk. log: VecDeque, error: Option, } pub struct BuildState { /// What this project produces, in the order it produces it. A run walks /// them, and the phase names on the status are the component names, so /// the card shows which one is being worked on without any new /// machinery. components: Vec, /// Which app this is, for naming the log files a build writes. The /// only thing here that needs the key; everything else is about the /// project on disk. key: String, /// Whether the Pull button is offered. Per project rather than per /// component: there is one checkout. git_pull: bool, project_path: PathBuf, /// True for the entry that *is* this server. /// /// Only its `Server` component is delivered differently: that component /// is this process, so restarting it means exec-ing the binary just /// built (`crate::restart`) rather than asking the service script to /// stop and start the code doing the asking -- which would kill the /// walk partway through, with nobody left to report that it finished. /// /// Passed in rather than read from the declaration because no project /// may claim it: a file that could would be a file that can make this /// server exec an arbitrary binary. is_self: bool, /// Force git's remote commands onto IPv4, for the fetch a pull starts. /// Beside `git_pull` and treated the same way, including in /// [`Self::matches`] -- toggling it has to reach the next pull, and /// the state is reused across config writes unless something says not /// to. git_ipv4: bool, /// Shared with the list, so a pull can record that this checkout is /// current instead of leaving a stale "new commits" on the card. shared: Shared, inner: Mutex, } /// What `/status` answers: whether anything needs building, and what is /// being done about it. /// /// One struct rather than a nested one. It was split so a card on /// `/manifest` could carry the run half without paying for `stale`, which /// walks every component's directory -- but nothing on the phone ever read /// that field, so the split was an indirection with one user and the /// second half of it has gone. #[derive(Serialize)] #[serde(rename_all = "camelCase")] pub struct BuildStatus { /// The expensive one: answering it walks every component's directory, /// which is why this is on `/status` and not on the manifest path. pub stale: bool, /// Anything at all is happening for this project -- a pull, or any /// component being built. Deliberately the *project's* question, for /// the controls that act on the whole checkout; a caller asking about /// one component reads that component's `step` instead, because this /// answers yes while a sibling builds and would disable a button that /// has nothing to wait for. pub building: bool, /// Why the last pull could not be made. Pulls only: a build failure is /// reported against the component that produced it, since two /// building at once have two outcomes and this is one field. pub error: Option, /// That failure was a pull the checkout has no fast-forward for, /// because it shares no history with its upstream. The phone offers /// the forced pull for this and nothing else. pub unrelated_histories: bool, /// What the whole project is doing -- fetching, pulling -- absent /// between runs and while the work belongs to a component instead. pub phase: Option, /// Every component built since this server started, in the order they /// were first reached. The card draws each of these inside that /// component's own row, and reads whether *it* is busy from its own /// entry. pub components: Vec, } /// One component's part of a build, as the phone sees it. #[derive(Serialize, Clone)] #[serde(rename_all = "camelCase")] pub struct ComponentStatus { pub name: String, /// What it is doing now, absent once it has finished. #[serde(skip_serializing_if = "Option::is_none")] pub step: Option, /// Steps done and steps total, when the running command reports them. /// Absent for a command that says nothing, which is most of them -- /// the phone shows a bar that only spins rather than inventing a /// number. #[serde(skip_serializing_if = "Option::is_none")] pub progress: Option, /// The last few lines this component printed, oldest first. pub log: Vec, /// Why this component's step failed, if it did. #[serde(skip_serializing_if = "Option::is_none")] pub error: Option, } /// Reads a progress report off a line of build output, in any of the /// shapes this understands. /// /// Anything that isn't exactly that shape is output, not a report: a build /// that prints the marker with something unparseable after it has a bug /// worth seeing in its log rather than being silently swallowed. fn parse_progress(line: &str) -> Option<(u64, u64)> { let line = line.trim(); if let Some(rest) = line.strip_prefix(PROGRESS_MARKER) { let (done, total) = rest.trim().split_once('/')?; return Some((done.trim().parse().ok()?, total.trim().parse().ok()?)); } parse_cargo_progress(line) } /// Reads cargo's own counter: `Building [===> ] 236/237: crate(bin)`. /// /// Cargo's, not this server's arithmetic -- it is the number of units it /// has left to compile, which is exactly the thing a bar can honestly be /// drawn from. `spawn` is what makes cargo emit it into a pipe at all. /// /// Anchored on the bracketed bar so that a line merely containing `12/34` /// is not read as progress: build output is full of numbers. fn parse_cargo_progress(line: &str) -> Option<(u64, u64)> { let rest = line.strip_prefix("Building ")?; let counts = rest.strip_prefix('[')?.split_once(']')?.1.trim(); // `236/237: dev-updater(bin)` -- the crate after the colon is what it // is working on now, which the log line above it already said. let counts = counts.split(':').next()?; let (done, total) = counts.split_once('/')?; Some((done.trim().parse().ok()?, total.trim().parse().ok()?)) } /// How much command output is kept for the phone. Enough to see what a /// build is doing, far short of keeping a build log in memory. const LOG_LINES: usize = 40; /// The step a component is in while its own build command runs, named /// once because two places have to agree on it: `claim` writes it when it /// takes the component, and `build_component` writes it again when it /// actually starts. const BUILDING: &str = "building"; /// Calls `emit` once per segment of a command's output, where a segment /// ends at a newline **or** a carriage return, and once more for anything /// left unterminated when the stream ends. /// /// Both terminators, because a tool that redraws a counter in place ends /// its line with `\r` and no newline. Cargo does exactly this: /// /// ```text /// Compiling dev-updater v0.1.0 (...)\n /// Building [=====> ] 236/237: dev-updater(bin) \r Finished `release` ...\n /// ``` /// /// `BufReader::lines()` splits on `\n` alone, so the count sat unread /// until whatever printed the next newline. On a build with many crates /// to compile that is the next `Compiling` line and the bar moved fine; /// on an incremental one -- a single dirty crate, which is what a pull /// produces -- it is the `Finished` line, arriving after the build is /// over. So the card showed nothing to measure for exactly the builds /// somebody watches, and it looked like a build system that could not /// report progress rather than a report being held. /// /// A `\r\n` pair ends a segment and then opens an empty one, which the /// caller drops along with every other blank segment. fn read_segments(pipe: impl std::io::Read, mut emit: impl FnMut(&str)) { use std::io::BufRead; let mut reader = std::io::BufReader::new(pipe); // Held across fills, because a terminator is what ends a segment and // a read can stop anywhere -- including inside a multi-byte // character, which is safe here only because both terminators are // ASCII and so cannot appear inside one. let mut pending: Vec = Vec::new(); loop { // Copied out so the borrow of the reader ends before `consume`. let chunk = match reader.fill_buf() { Ok([]) => break, Ok(bytes) => bytes.to_vec(), // The process's exit status is what says whether the build // worked; a pipe that cannot be read further is just the end // of what there is to show. Err(_) => break, }; reader.consume(chunk.len()); for byte in chunk { if byte == b'\n' || byte == b'\r' { emit(&String::from_utf8_lossy(&pending)); pending.clear(); } else { pending.push(byte); } } } if !pending.is_empty() { emit(&String::from_utf8_lossy(&pending)); } } impl BuildState { pub fn new( key: String, components: Vec, git_pull: bool, git_ipv4: bool, project_path: PathBuf, shared: Shared, is_self: bool, ) -> Arc { let built_from = components .iter() .filter_map(|component| { Some(( component.name().to_string(), component.built_from()?.to_string(), )) }) .collect(); Arc::new(Self { key, inner: Mutex::new(Inner { built_from, ..Inner::default() }), components, git_pull, git_ipv4, project_path, shared, is_self, }) } /// Whether this state was built from the same configuration, i.e. /// whether it can be carried across a config reload -- see /// `crate::registry::build_entries`. /// Compared by *declaration*, not by full equality. Recording what a /// build was made from writes the config, which rebuilds the entry /// list -- and if that counted as a different configuration the state /// would be replaced mid-walk, leaving the phone polling a fresh /// `BuildState` that reports nothing is building. The same is true of /// the package a download reads back. pub fn matches(&self, components: &[Component], git_pull: bool, git_ipv4: bool) -> bool { self.git_pull == git_pull && self.git_ipv4 == git_ipv4 && self.components.len() == components.len() && std::iter::zip(&self.components, components).all(|(mine, theirs)| { // The declaration, plus the one choice that changes what // this state would actually run. `same_declaration` // deliberately ignores the chosen mode -- picking one is // not the project asking for something new -- but this // state holds a *snapshot* of the components it builds // from, so a mode changed underneath it would go on // running the old mode's command from a card reporting // the new one. Strip is not here: it is decided at // download, and nothing this state does depends on it. mine.same_declaration(theirs) && mine.effective_mode() == theirs.effective_mode() }) } /// Idempotent: kicks off the build in the background if this app is /// stale and no build is already running, a no-op otherwise. Safe for /// the phone to call on every download, which is exactly what it does. /// /// `component` narrows both the staleness check and the build itself /// to one component -- what the phone asks before downloading it, so /// that pressing Update on one client of a multi-client project /// doesn't also pay for the other's build. `None` is every other /// caller, which still means "the whole project". pub fn trigger_if_needed(self: &Arc, component: Option<&str>, record: RecordBuilt) { // Before the lock, not inside it -- see `is_stale`. if self.is_stale(component) { self.build_now(component, record); } } /// Builds whether or not anything looks stale, for somebody who asked. /// /// The staleness rules exist to keep a *download* from rebuilding the /// world; they have no business overruling a person who pressed a /// button. This is also what makes the freshness signal reachable at /// all: the commit a build was made from is only recorded by a build /// that runs here, so a project already current with its checkout /// could never record one and would report unknown for ever. /// /// Idempotent per *component*, not per project: a component already /// being built is left alone, and every other one this selects starts /// regardless. So two components of one project build at the same /// time, and asking for one says nothing about the others -- which is /// the whole point of a project being able to produce more than one /// thing. The phone notices a component is busy by polling `/status` /// and reading that component's own step. /// /// `component` restricts the run to one named component, or every one /// with a command for `None` -- see [`Self::trigger_if_needed`] for /// why a caller would want the former. pub fn build_now(self: &Arc, component: Option<&str>, record: RecordBuilt) { let claimed = { let mut inner = self.inner.lock().unwrap(); // The one thing a component still waits behind. See // `Inner::pulling`. if inner.pulling { return; } self.claim(&mut inner, component) }; self.start(claimed, record); } /// Marks each component `name` selects that has something to build /// and is not already building, and answers where they are. /// /// Takes the guard rather than the lock so that a caller can do this /// and something else in one step -- which the pull needs: releasing /// `pulling` and claiming what it decided to build have to happen /// together, or a phone polling in between sees a project that is /// neither pulling nor building and reads its run as finished. /// /// The entry is written **here**, synchronously, rather than by the /// build thread once it gets going. The route answers as soon as this /// returns, so a status read in that window would otherwise find the /// component idle -- and idle is exactly what the phone is waiting /// for. fn claim(&self, inner: &mut Inner, name: Option<&str>) -> Vec { let mut claimed = Vec::new(); for (index, component) in self.components.iter().enumerate() { if name.is_some_and(|name| component.name() != name) { continue; } if component.build().is_empty() { continue; } if inner.component_running(component.name()) { continue; } // Whatever the last run left against this component goes now, // so nothing from it can be read as this run's outcome -- an // old error in particular, which the phone treats as a reason // not to download. reset_run(inner, component.name()); claimed.push(index); } if !claimed.is_empty() { // The pull this describes is the one being built on top of. inner.error = None; inner.unrelated_histories = false; } claimed } /// Runs what [`Self::claim`] claimed, off the request's thread. fn start(self: &Arc, claimed: Vec, record: RecordBuilt) { if claimed.is_empty() { return; } let this = Arc::clone(self); tokio::task::spawn_blocking(move || this.run_claimed(claimed, &record)); } /// Whether any component `name` selects is behind. Whole project for /// `None`, which is what a run with nothing named means: one /// component being stale is reason enough to walk the whole list, and /// each component's own command is skipped if it has nothing to do. fn is_stale(&self, name: Option<&str>) -> bool { // Read once, here, and passed down. `component_is_stale` must not // take this lock itself: it is reached from callers that already // hold it, and a `std::sync::Mutex` is not reentrant, so doing so // is a deadlock rather than a slow path. let built_from = self.inner.lock().unwrap().built_from.clone(); self.components .iter() .filter(|component| name.is_none_or(|name| component.name() == name)) .any(|component| self.component_is_stale(component, &built_from)) } /// What can be said about this component's build, for the card. /// /// Deliberately not the same question as [`Self::component_is_stale`], /// which decides whether to *run* a build. Uncommitted work makes the /// comparison unreliable, so it reports unknown -- but it must not /// trigger a rebuild: somebody editing on the build machine is /// building it themselves, and this server's job in that state is to /// notice the new output rather than to produce one. Rebuilding /// uncommitted work and sending it to a phone would also be a /// surprising thing to do with work its author has not committed. pub fn freshness(&self, component: &Component) -> Freshness { // Behind for a reason no commit can express: what is on disk was // built some other way than this component is set to build now. // Reported before the checkout is consulted at all, because it is // a fact rather than a comparison -- a clean tree at the very // commit the debug build was made from still does not make that // build a release one. if component.built_in_another_mode() { return Freshness::Behind; } let built = self .inner .lock() .unwrap() .built_from .get(component.name()) .cloned(); let Some(built) = built else { return Freshness::Unknown; }; if crate::git::subtree_dirty(&self.project_path, &component.watched_paths()) != Some(false) { return Freshness::Unknown; } match crate::git::subtree_head(&self.project_path, &component.watched_paths()) { Some(current) if current == built => Freshness::Current, Some(_) => Freshness::Behind, None => Freshness::Unknown, } } fn component_is_stale( &self, component: &Component, built_from: &HashMap, ) -> bool { if component.build().is_empty() { return false; } // The same fact `freshness` reports as Behind, and it has to be // asked here too: for an `Apk`, the "never built at all" check // below finds *any* variant under the component's directory, so a // debug build sitting there is enough to make a component // switched to release look built. Nothing else would notice -- // the commit has not moved -- so Update would do nothing and the // phone would install the debug APK from a card saying release. if component.built_in_another_mode() { return true; } // Nothing built at all is as far behind as an output gets, and it // is checked before any rule rather than after: a `staleWhen` // compares two files *inside* a build, so a component that has // never been built reports not-stale under it and would leave the // first build impossible to trigger -- which a project can be // added before having done. // // Scoped to *this component's own* directory, not the project // root: a project producing two APKs has one component's output // sitting under the root-anchored patterns too (an inner // `*/build/outputs/apk/*/*.apk` matches a one-level subdirectory // regardless of which component it belongs to), which made the // whole project read as "something is built here" the moment // either component had ever been built -- masking that the // *other* component, never built, had nothing to trigger it. // // Only asked of an `Apk`: a `Server` never has one to find under // its own directory by definition, so the same question asked of // it would report every server "never built" for ever, which is // exactly the false staleness this check exists to rule out. if matches!(component, Component::Apk { .. }) && crate::discover::find_apks(&component.dir(&self.project_path)).is_empty() { return true; } // Behind the checkout: this component was built from a commit // that its own directory has since moved past. The case nothing // used to catch -- a pull that fetched but did not build, because // the declaration was waiting to be accepted or because the walk // failed earlier -- left an output older than the commits it // should have come from, and the card had no reason to doubt it. // // Both halves must be known. No recorded commit means this server // has never built it, and an unreadable checkout means it cannot // tell; neither is evidence of staleness, and announcing one would // make every project built before this existed demand a rebuild. if let Some(built) = built_from.get(component.name()) && crate::git::subtree_head(&self.project_path, &component.watched_paths()) .is_some_and(|current| ¤t != built) { return true; } // Deliberately not the other way round: a matching commit does not // prove currency, because a `staleWhen` rule compares two build // outputs and can be behind for reasons no commit explains. component .stale_when() .is_some_and(|rule| rule.is_stale(&self.project_path)) } /// Fetches, fast-forwards if there is anything to take, and builds -- /// the Pull button, which acts on the build machine rather than the /// phone. While one is running, another does nothing. /// /// Building happens when the pull actually moved the branch, or when /// the configured staleness rule says the output is behind anyway; a /// pull that finds nothing new therefore costs nothing. /// /// `may_build` answers "may the project's command run?" -- false when /// it is asking for a build step nobody has accepted (see /// `AppEntry::pending_declaration`), in which case this pulls and /// stops. The decision belongs to the caller because it is about the /// project's config file, which is not this module's business -- here /// it is simply "don't run the command". /// /// It is a closure rather than a bool because of *when* it has to be /// asked: **after the fast-forward, not before it.** The declaration /// lives in the checkout, so the pull is the one thing that changes /// the answer, and a bool computed at the call site is necessarily the /// answer for the commit being replaced. Passing one meant a pull /// whose new commits changed the declaration built anyway -- the card /// went off and updated the app instead of stopping to ask, which is /// the single case this gate exists for. /// `force` abandons this checkout's own history in favour of the /// upstream's, and is only ever pressed after a pull has already /// reported that the two are unrelated (`git::PullError`). It is a /// parameter rather than something decided here because it is a /// person's answer to that report, not a state of the repository. /// /// Unlike a build, this is exclusive with everything the project is /// doing: it rewrites the one checkout every component is built from, /// so it waits for any component still building and blocks any that /// would start. That is the project's own coupling rather than the /// components', and it is the only one left here. pub fn pull_and_build( self: &Arc, force: bool, may_build: impl Fn() -> bool + Send + 'static, record: RecordBuilt, ) { self.after_moving(move |this| this.pull(force), may_build, record); } /// Moves this checkout onto `target` -- a branch or a commit -- and /// then builds whatever that left behind. /// /// Exclusive with everything, and deferred to the same machinery as a /// pull, because it is the same kind of act: there is one checkout, /// and this rewrites the files every component builds from. It /// reports through the project's own state for the same reason, so a /// card shows it exactly as it shows a pull. pub fn checkout_and_build( self: &Arc, target: String, may_build: impl Fn() -> bool + Send + 'static, record: RecordBuilt, ) { self.after_moving( move |this| { this.begin_project("checking out"); crate::git::checkout(&this.project_path, &target)?; tracing::info!("checked out {target} in {}", this.project_path.display()); // Always "something moved": a checkout onto the commit // that was already there is the one case this over-reports, // and the cost of that is one build that had nothing to do. // Under-reporting costs a checkout whose outputs stay from // the commit before it, which is the silent kind. Ok(true) }, may_build, record, ); } /// The half a pull and a checkout share: run the thing that moves the /// working tree, then build what it left behind. /// /// One copy, because the part that is easy to get wrong is not the /// git command -- it is handing the run over under a single lock so /// that nothing can observe the moment between the move ending and /// the builds it decided on starting. A phone polling in that gap /// sees a project that is neither pulling nor building and calls the /// run finished. fn after_moving( self: &Arc, move_it: impl FnOnce(&Arc) -> Result + Send + 'static, may_build: impl Fn() -> bool + Send + 'static, record: RecordBuilt, ) { // Claimed before anything is spawned, and while nothing else is // running: one checkout, so a move is exclusive with every build // as well as with another move. { let mut inner = self.inner.lock().unwrap(); if inner.anything_running() { return; } inner.pulling = true; inner.error = None; inner.unrelated_histories = false; } let this = Arc::clone(self); tokio::task::spawn_blocking(move || { let outcome = move_it(&this); match outcome { Err(error) => { tracing::error!("moving the checkout failed: {}", error.message); this.fail_pull(error); } Ok(pulled) => { // Nothing configured to build, or nothing allowed to: // the pull was the whole job. // `may_build()` is called here, with the pulled // declaration on disk, for the reason in the doc // comment above. Both it and `is_stale` take the lock // themselves, so they are asked before it is held. let build = may_build() && this.has_command() && (pulled || this.is_stale(None)); // Handing the run over in one step, so nothing can // observe the moment between the pull ending and the // builds it decided on starting -- see `claim`. let claimed = { let mut inner = this.inner.lock().unwrap(); let claimed = match build { true => this.claim(&mut inner, None), false => Vec::new(), }; inner.pulling = false; inner.phase = None; claimed }; this.run_claimed(claimed, &record); } } }); } /// Fetches and fast-forwards, reporting whether anything arrived. fn pull(&self, force: bool) -> Result { self.begin_project("fetching"); crate::git::fetch(&self.project_path, self.git_ipv4)?; // Counted after the fetch, from refs now on disk: this is the one // place that has the objects to count against. let behind = crate::git::behind(&self.project_path); if behind == 0 { return Ok(false); } self.begin_project("pulling"); crate::git::pull(&self.project_path, self.git_ipv4, force)?; tracing::info!( "pulled {behind} commit(s) into {}", self.project_path.display() ); // The card should stop offering what was just taken, rather than // waiting out the check interval to notice. self.shared.remote_checks.mark_current(&self.project_path); Ok(true) } /// Builds the claimed components, at once, and reports each against /// itself. /// /// Every one of them together rather than in turn -- they are /// independent, a Rust build and a Gradle build share nothing but the /// machine, and measured on this one, running them together takes /// about three quarters of the time running them in turn does. /// /// What running them together costs is that a failure no longer stops /// the others: they are already running by the time it happens, so /// stopping them would mean killing work that is probably fine. There /// is no "the run failed" left to report either way -- an outcome /// belongs to the component that produced it, and the phone reads it /// off that component's own entry. fn run_claimed(self: &Arc, claimed: Vec, record: &RecordBuilt) { let mut running = Vec::new(); for index in claimed { let this = Arc::clone(self); let record = Arc::clone(record); running.push(( self.components[index].name().to_string(), std::thread::spawn(move || this.build_indexed(index, &record)), )); } // Whether anything in *this* run fell over, which is only asked // about the restart below. It is not reported anywhere: the // component that failed has already recorded its own message. let mut failed = false; // Set by the component that is this process; acted on once every // other component in the run has finished. See `is_self`. let mut restart_self = false; for (name, handle) in running { let outcome = match handle.join() { // A panic in a build thread is this server's bug, not the // project's, but the card still has to say something -- // silence would read as a build that simply did nothing. // Recorded here because the panic is what stopped // `build_indexed` from doing it, and a component left with // an open step never stops looking busy. Err(_) => { let message = format!("building {name} panicked -- see this server's log"); self.finish_component(&name, Some(message.clone())); Err(message) } Ok(outcome) => outcome, }; match outcome { Ok(is_self) => restart_self |= is_self, Err(_) => failed = true, } } // Not restarting on a failure is about *this process*, not about // the components: an exec drops the connection the phone is // reading the failure over, and it would lose the report it is // waiting for. let mut restart = !failed && restart_self; // A build with nothing to do leaves the binary alone, and exec-ing // into the same file would drop the phone's connection to deliver // the build it already had. if restart && crate::restart::binary_unchanged() { tracing::info!( "the build left this server's binary untouched, so it is not restarting" ); restart = false; } if restart { // Every component has closed its own entry by now, so whatever // happens next cannot take the report away from whoever asked // for it. crate::restart::deferred(self.handover(), Arc::clone(&self.shared.downloads)); } } /// Builds the component at `index`, and closes its entry whatever /// happens. /// /// By index because the components belong to this state, which the /// thread holds an `Arc` to -- borrowing one across a thread boundary /// would tie the thread's lifetime to this call's. /// /// Answers whether that component is this server itself, which is the /// one thing the caller has to act on after every thread is done. fn build_indexed(self: &Arc, index: usize, record: &RecordBuilt) -> Result { let component = &self.components[index]; let outcome = self.build_component(component, record); // One place, so no exit from `build_component` can leave a // component looking like it is still working. self.finish_component(component.name(), outcome.as_ref().err().cloned()); outcome } /// Builds one component and delivers it, or says why it could not. /// /// One function rather than the walk's body inline, so that "the build /// stopped here" is recorded in exactly one place -- every failure /// leaves by the same return, and the caller names the component /// without each site having to remember to. fn build_component( self: &Arc, component: &Component, record: &RecordBuilt, ) -> Result { self.begin_component(component.name(), BUILDING); if let Err(message) = self.run_streaming(component) { tracing::error!("{} failed: {message}", component.name()); return Err(message); } // Recorded per component and only on success, so a walk that // fails partway credits what actually built and nothing else: // if the APK fails after the server succeeded, the server is // current and the APK is not. Scoped to the component's own // directory, so a commit touching only the app does not make // the server look stale. // // A checkout that cannot be read records nothing, which reads // as unknown rather than as current -- see `component_is_stale`. if let Some(sha) = crate::git::subtree_head(&self.project_path, &component.watched_paths()) { self.inner .lock() .unwrap() .built_from .insert(component.name().to_string(), sha.clone()); record(component.name(), sha); } // Delivering a server is restarting it, so it happens here rather // than at the end: a later component's failure should not leave a // built-but-not-restarted server behind. let Component::Server { name, .. } = component else { return Ok(false); }; let Some(service) = crate::service::driver(&self.key, component) else { return Ok(false); }; let service = &service; // What the manager says decides both halves: whether the unit is // worth refreshing, and whether anything should be restarted // afterwards. let installed = match crate::service::status(service, &self.project_path, component.cwd()) { Ok(state) => Some(state), // Not a build failure: the build worked, and whether the // service came back is a separate thing the card reports on // its own. Err(err) => { tracing::warn!("could not ask {name} how it is running: {err}"); None } }; // Refresh the unit whenever there is one, for either kind of // server. It is generated from a script in the repository, so a // pull can change how the service is *defined* and not only what // it runs -- and that change has to land even when the build // produced an identical binary, which is the ordinary case for a // pull that only touched the service script. `start.sh` has // always reinstalled every run for exactly this reason. // // Safe on a running service: writing the unit and reloading stops // nothing. if installed.is_some_and(|state| state != crate::service::ServiceState::NotInstalled) { self.begin_component(name, "installing"); if let Err(message) = self.run_streaming_command(name, service, component.cwd(), &["install"], None) { tracing::error!("installing {name} failed: {message}"); return Err(message); } } if self.is_self { // This one is the process doing the walking, so its restart // waits until every other component is done and the result is // reported. See the end of `run_build`. return Ok(true); } if installed == Some(crate::service::ServiceState::Running) { self.begin_component(name, "restarting"); if let Err(message) = self.run_streaming_command(name, service, component.cwd(), &["restart"], None) { tracing::error!("restarting {name} failed: {message}"); return Err(message); } } else if let Some(state) = installed { tracing::info!("{name} is {state:?}, so it was not started"); } Ok(false) } /// How this server should be restarted, if it declares a way. /// /// Only ever asked for this server's own project, where exactly one /// `Server` component is this process. fn handover(&self) -> Option { let (component, script) = self.components.iter().find_map(|component| { matches!(component, Component::Server { .. }) .then(|| crate::service::driver(&self.key, component)) .flatten() .map(|script| (component, script)) })?; Some(crate::restart::Handover { script, project: self.project_path.clone(), cwd: component.cwd().map(std::path::Path::to_path_buf), }) } /// Runs the build command, recording its output as it arrives rather /// than collecting it at the end. /// /// A build is the long part of a pull, and a phone showing nothing but /// a spinner for a minute is indistinguishable from one that is stuck. /// Reading the pipes as they fill also means a chatty build can't fill /// a pipe buffer and block itself, which collecting output at the end /// risks for anything verbose. fn run_streaming(self: &Arc, component: &Component) -> Result<(), String> { // Opened per component and rotated here, so a failed build is // still readable after this server restarts -- the tail kept in // memory for the progress display dies with the process, and a // build that brought the server down is exactly the one somebody // wants to read afterwards. let log = crate::logs::open_build_log(&self.key, component.name()) .map(|file| Arc::new(Mutex::new(file))); // The mode, for a build command written once -- which is how a // project with a script that takes `release` or `debug` says it, // and why nothing has to be written twice in that case. Nothing // for a command written per mode: that command already *is* the // answer, and handing the word over as well would pass a stray // argument to a build that never asked for one. let mode = component.build_mode_argument(); self.run_streaming_command( component.name(), component.build(), component.cwd(), mode.as_slice(), log, ) } fn run_streaming_command( self: &Arc, component: &str, command: &crate::config::Command, cwd: Option<&std::path::Path>, extra: &[&str], log: Option>>, ) -> Result<(), String> { let mut child = self .spawn(command, cwd, extra)? .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::piped()) .spawn() .map_err(|err| format!("failed to start the build: {err}"))?; // Both pipes, on separate threads: whichever the build writes to, // waiting on only one of them would deadlock when the other fills. let mut readers = Vec::new(); for pipe in [ child .stdout .take() .map(|out| Box::new(out) as Box), child .stderr .take() .map(|err| Box::new(err) as Box), ] .into_iter() .flatten() { let state = Arc::clone(self); let log = log.clone(); // Owned per reader thread, because the line it routes has to // reach the component that printed it and the thread outlives // this borrow. let owned = component.to_string(); readers.push(std::thread::spawn(move || { read_segments(pipe, |segment| { let segment = segment.trim_end(); if segment.is_empty() { return; } // The file gets everything, including the progress // markers: it is the record of what the command // actually printed, not the display. One segment per // line, which is already what a wrapper emitting // `@@progress` produces, and what keeps a line from // one pipe out of the middle of a line from the other. if let Some(log) = &log { use std::io::Write; let _ = writeln!(log.lock().unwrap(), "{segment}"); } match parse_progress(segment) { // Machinery, not output: it would only push the // build's own last line out of the display. Some(progress) => state.set_progress(&owned, progress), None => state.log_line(&owned, segment.to_string()), } }); })); } let status = child .wait() .map_err(|err| format!("waiting on the build: {err}"))?; for reader in readers { let _ = reader.join(); } if status.success() { return Ok(()); } // The tail is what a failing build explains itself in, and it is // already here rather than needing to be collected separately. let tail: Vec = self .inner .lock() .unwrap() .runs .iter() .find(|run| run.name == component) .map(|run| run.log.iter().cloned().collect()) .unwrap_or_default(); Err(format!( "the build step failed ({status}):\n{}", truncate_tail(&tail.join("\n"), 2000) )) } /// The command to run, configured but not started. fn spawn( &self, command: &crate::config::Command, cwd: Option<&std::path::Path>, extra: &[&str], ) -> Result { let mut process = command.to_process(&self.project_path, cwd, extra)?; // Cargo hides its progress counter unless it is talking to a // terminal, and it is talking to a pipe. Asked for it explicitly, // it prints `Building [===> ] 236/237: crate` -- counts it worked // out itself, which is the only kind this shows. Set here rather // than asked of each project, because a build that happens to be // cargo should not have to know that it is being watched, and // because a project could otherwise only offer this by // reimplementing it. // // Harmless to anything that is not cargo: an unread variable. // Width fixed so the bar is not sized against a terminal there // isn't; nothing reads the bar itself, only the numbers after it. process.env("CARGO_TERM_PROGRESS_WHEN", "always"); process.env("CARGO_TERM_PROGRESS_WIDTH", "100"); // For the build systems that cannot be asked, a wrapper this // server ships. Handed over as a path rather than expected on // PATH, and absent when a person runs the same script by hand -- // which is why a script tests for it rather than depending on it. // See `crate::shipped`. process.env("DEV_UPDATER_PROGRESS", crate::shipped::build_progress()); Ok(process) } /// Marks what the *project* is doing. Only fetching and pulling: the /// rest belongs to a component. fn begin_project(&self, name: &str) { self.inner.lock().unwrap().phase = Some(name.to_string()); } /// Marks one component as doing `step`. /// /// Its entry already exists -- `claim` writes it before the build /// thread starts, so that nothing can read the component as idle in /// between -- but it is created here if it somehow does not, since a /// step nobody can see is worse than a duplicated one. fn begin_component(&self, component: &str, step: &str) { let mut inner = self.inner.lock().unwrap(); // The project-wide phase is over once a component is working; // otherwise the card shows "pulling" under a build. inner.phase = None; match inner.runs.iter_mut().find(|run| run.name == component) { Some(run) => { run.step = Some(step.to_string()); // A count belongs to the step that printed it. run.progress = None; } None => reset_run(&mut inner, component), } } /// Marks one component as done, with why it stopped if it failed. /// /// The end of a run, now that a run is one component's: there is no /// project-wide "finished" left to say, and the phone reads this /// component's closed step as the answer for this component alone. fn finish_component(&self, component: &str, error: Option) { let mut inner = self.inner.lock().unwrap(); if let Some(run) = inner.runs.iter_mut().find(|run| run.name == component) { run.step = None; run.error = error; } } /// Ends a pull that could not be made. /// /// The project's own failure rather than a component's, and it /// carries the one thing a build failure cannot: whether abandoning /// this checkout's own history would clear it. Cleared at the start of /// anything that runs afterwards, so a pull is the only thing that can /// ever make it true. fn fail_pull(&self, error: crate::git::PullError) { let mut inner = self.inner.lock().unwrap(); inner.unrelated_histories = error.unrelated_histories; inner.error = Some(error.message); inner.phase = None; inner.pulling = false; } /// Records how far the running command says it has got. Replaces the /// previous count rather than accumulating: the command reports its /// own total, and `begin` clears this so one component's count is /// never shown against the next one's work. fn set_progress(&self, component: &str, progress: (u64, u64)) { let mut inner = self.inner.lock().unwrap(); if let Some(run) = inner.runs.iter_mut().find(|run| run.name == component) { run.progress = Some(progress); } } /// Records one line of a component's output for its progress display, /// dropping the oldest once the window is full. /// /// Kept against the component that printed it rather than in one /// stream, so two building at once do not interleave into something /// neither of them said. fn log_line(&self, component: &str, line: String) { let mut inner = self.inner.lock().unwrap(); if let Some(run) = inner.runs.iter_mut().find(|run| run.name == component) { if run.log.len() == LOG_LINES { run.log.pop_front(); } run.log.push_back(line); } } /// Whether there is anything to run, as opposed to a checkout that /// can only be pulled. pub fn has_command(&self) -> bool { crate::config::has_command(&self.components) } /// Whether this component's last build stopped at it. /// /// The card asks so it can open that component's build log first. A /// component that has not failed answers false, which is what makes /// the runtime log the ordinary default -- and it is asked of the /// component's own entry, so one component's failure cannot send /// another's card to the wrong tab. pub fn build_failed(&self, component: &str) -> bool { self.inner .lock() .unwrap() .runs .iter() .any(|run| run.name == component && run.error.is_some()) } pub fn status(&self) -> BuildStatus { // Before the lock, not inside it -- see `is_stale`. let stale = self.is_stale(None); let inner = self.inner.lock().unwrap(); BuildStatus { stale, building: inner.anything_running(), error: inner.error.clone(), unrelated_histories: inner.unrelated_histories, phase: inner.phase.clone(), components: inner .runs .iter() .map(|run| ComponentStatus { name: run.name.clone(), step: run.step.clone(), progress: run .progress .filter(|(_, total)| *total > 0) .map(|(done, total)| Progress { done: done.min(total), total, }), log: run.log.iter().cloned().collect(), error: run.error.clone(), }) .collect(), } } } /// Starts this component's entry over, replacing whatever the last build /// of it left behind. /// /// The step is set here rather than by the thread that will do the work, /// so there is no window in which a claimed component reads as idle. The /// word is the first thing every build does anyway (`build_component`), /// so nothing has to say it twice. fn reset_run(inner: &mut Inner, component: &str) { let fresh = ComponentRun { name: component.to_string(), step: Some(BUILDING.to_string()), progress: None, log: VecDeque::new(), error: None, }; match inner.runs.iter_mut().find(|run| run.name == component) { // In place, so the order the phone receives components in stays // the order it first saw them. Some(run) => *run = fresh, None => inner.runs.push(fresh), } } fn truncate_tail(s: &str, max_chars: usize) -> String { let char_count = s.chars().count(); if char_count <= max_chars { s.to_string() } else { s.chars().skip(char_count - max_chars).collect() } } #[cfg(test)] mod tests { /// Cargo's own counter, which `spawn` asks it to print into a pipe. /// The numbers are cargo's; nothing here works them out. #[test] fn cargos_progress_counter_is_read_and_other_numbers_are_not() { assert_eq!( parse_progress(" Building [=====> ] 236/237: dev-updater(bin)"), Some((236, 237)) ); assert_eq!(parse_progress(" Building [] 0/12"), Some((0, 12))); // Build output is full of numbers; only the bracketed bar means a // count. A crate that happens to be called this must not move it. assert_eq!(parse_progress(" Compiling foo v1.2.3 (12/34)"), None); assert_eq!(parse_progress("warning: 3/4 of these are unused"), None); assert_eq!(parse_progress("Building the thing 1/2"), None); } /// A project can still report its own, which is the only way a tool /// with no counter of its own gets a bar. #[test] fn the_explicit_marker_still_wins() { assert_eq!(parse_progress("@@progress 3/7"), Some((3, 7))); assert_eq!(parse_progress("@@progress nonsense"), None); } use std::path::Path; use super::*; fn run(dir: &Path, args: &[&str]) { let out = std::process::Command::new(args[0]) .args(&args[1..]) .current_dir(dir) .output() .expect("run"); assert!( out.status.success(), "{args:?}: {}", String::from_utf8_lossy(&out.stderr) ); } /// One checkout producing two things, which is the shape both real /// projects have: a server under `server/` and an app under `app/`. fn two_component_checkout(root: &Path) -> Vec { std::fs::create_dir_all(root.join("server")).expect("mkdir"); std::fs::create_dir_all(root.join("app/build/outputs/apk/debug")).expect("mkdir"); // Something for `find_apks` to see, so staleness is decided by the // rules under test rather than by nothing being built. std::fs::write(root.join("app/build/outputs/apk/debug/a.apk"), b"").expect("write"); std::fs::write(root.join("server/main.rs"), "one").expect("write"); std::fs::write(root.join("app/main.kt"), "one").expect("write"); for args in [ &["git", "init", "-q", "-b", "main"][..], &["git", "config", "user.email", "t@example.com"], &["git", "config", "user.name", "Test"], &["git", "add", "-A"], &["git", "commit", "-qm", "one"], ] { run(root, args); } vec![ Component::Server { name: "backend".to_string(), modes: Vec::new(), build: crate::config::ByMode::One(crate::config::Command::from_line("true")), cwd: Some(PathBuf::from("server")), stale_when: None, also_watch: Vec::new(), service: None, mode: None, built_from: None, built_mode: None, }, Component::Apk { name: "app".to_string(), modes: Vec::new(), build: crate::config::ByMode::One(crate::config::Command::from_line("true")), cwd: Some(PathBuf::from("app")), stale_when: None, also_watch: Vec::new(), strip: false, enroll: crate::config::Command::default(), strip_here: None, mode: None, package: None, built_from: None, built_mode: None, }, ] } /// A pull that brings a changed declaration must stop, not build. /// /// The gate answers "may this project's command run?", and the /// declaration it reads lives in the checkout -- so the pull is the /// one thing that changes the answer. Asked before the fast-forward /// it necessarily reports on the commit being replaced, which is how /// a card went off and updated the app instead of stopping to ask. /// Hence a closure: this holds the *moment* it is called. #[tokio::test] async fn a_pull_that_changes_the_declaration_does_not_build() { let dir = tempfile::tempdir().expect("tempdir"); let origin = dir.path().join("origin"); std::fs::create_dir_all(&origin).expect("mkdir"); let declare = |command: &str| format!("components: [Apk(name: \"app\", build: \"{command}\")],\n"); std::fs::write( origin.join(crate::config::PROJECT_CONFIG_FILE), declare("touch built-marker"), ) .expect("write"); for args in [ &["git", "init", "-q", "-b", "main"][..], &["git", "config", "user.email", "t@example.com"], &["git", "config", "user.name", "Test"], &["git", "add", "-A"], &["git", "commit", "-qm", "one"], ] { run(&origin, args); } let clone = dir.path().join("clone"); run( dir.path(), &[ "git", "clone", "-q", origin.to_str().unwrap(), clone.to_str().unwrap(), ], ); // What this machine accepted, which is what the checkout asks for // right now -- so the gate is open at the moment the button is // pressed, and only the pull closes it. let accepted = vec![Component::Apk { name: "app".to_string(), modes: Vec::new(), build: crate::config::ByMode::One(crate::config::Command::from_line( "touch built-marker", )), cwd: None, stale_when: None, also_watch: Vec::new(), strip: false, enroll: crate::config::Command::default(), strip_here: None, package: None, mode: None, built_from: None, built_mode: None, }]; let state = state_for(&clone, accepted.clone()); // The commit the pull is about to take, asking for a different // command from the one that was accepted. std::fs::write( origin.join(crate::config::PROJECT_CONFIG_FILE), declare("touch never-run"), ) .expect("write"); run(&origin, &["git", "commit", "-qam", "two"]); let project = clone.clone(); state.pull_and_build( false, move || { crate::config::project_config(&project) .declaration .matches_accepted(&accepted, None) }, Arc::new(|_: &str, _: String| {}), ); while state.status().building { tokio::time::sleep(std::time::Duration::from_millis(10)).await; } assert!( clone.join(".dev-updater.ron").exists(), "the pull itself still happens -- it is how the new request arrives to be read", ); assert!( !clone.join("built-marker").exists(), "the declaration changed under the acceptance, so nothing may run", ); } /// Naming one component builds only that one. /// /// The case a project with more than one component exists to avoid /// paying for: pressing Update on one client of a two-client project /// must not also run the other's build, which for a slow one (an ARM /// cross-compile, say) is the entire complaint a phone would have. #[tokio::test] async fn naming_a_component_builds_only_that_one() { let dir = tempfile::tempdir().expect("tempdir"); let root = dir.path(); std::fs::create_dir_all(root.join("server")).expect("mkdir"); std::fs::create_dir_all(root.join("app")).expect("mkdir"); let components = vec![ Component::Server { name: "backend".to_string(), modes: Vec::new(), build: crate::config::ByMode::One(crate::config::Command::from_line( "touch backend-built", )), cwd: Some(PathBuf::from("server")), stale_when: None, also_watch: Vec::new(), service: None, mode: None, built_from: None, built_mode: None, }, Component::Apk { name: "app".to_string(), modes: Vec::new(), build: crate::config::ByMode::One(crate::config::Command::from_line( "touch app-built", )), cwd: Some(PathBuf::from("app")), stale_when: None, also_watch: Vec::new(), strip: false, enroll: crate::config::Command::default(), strip_here: None, mode: None, package: None, built_from: None, built_mode: None, }, ]; let state = state_for(root, components); state.build_now(Some("app"), Arc::new(|_: &str, _: String| {})); let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); while state.status().building { assert!( std::time::Instant::now() < deadline, "build did not finish in time" ); tokio::time::sleep(std::time::Duration::from_millis(10)).await; } assert!( root.join("app/app-built").exists(), "the named component ran" ); assert!( !root.join("server/backend-built").exists(), "naming one component must not build the other", ); let status = state.status(); assert_eq!( status .components .iter() .map(|c| c.name.as_str()) .collect::>(), vec!["app"], "the status only reports the component that actually ran", ); } /// Building one component must neither block another's build nor make /// it look busy. /// /// The complaint this decoupling is for: with one build slot per /// project, pressing Update on one client of a two-client project /// left the other's request a silent no-op and its buttons disabled /// for the length of a build it had nothing to do with. Asserted from /// *inside* the slow component's build, because "both finished /// eventually" is equally true of running them one after the other -- /// what is being tested is that the second one does not wait. #[tokio::test] async fn one_component_building_does_not_hold_up_another() { let dir = tempfile::tempdir().expect("tempdir"); let root = dir.path(); for name in ["slow", "quick"] { std::fs::create_dir_all(root.join(name)).expect("mkdir"); } // A script rather than `sh -c '...'`: a command is split on // whitespace and never goes through a shell, so the quotes would // be four literal arguments and the "slow" build would fail // instantly -- which reads exactly like the two running in turn. let script = root.join("slow.sh"); std::fs::write(&script, "#!/bin/sh\nsleep 0.6\ntouch slow-built\n").expect("write"); std::fs::set_permissions(&script, std::os::unix::fs::PermissionsExt::from_mode(0o755)) .expect("chmod"); let components = ["slow", "quick"] .map(|name| Component::Apk { name: name.to_string(), modes: Vec::new(), build: crate::config::ByMode::One(crate::config::Command::from_line(match name { "slow" => "./slow.sh", _ => "touch quick-built", })), cwd: Some(PathBuf::from(name)), stale_when: None, also_watch: Vec::new(), strip: false, enroll: crate::config::Command::default(), strip_here: None, mode: None, package: None, built_from: None, built_mode: None, }) .to_vec(); let state = state_for(root, components); state.build_now(Some("slow"), Arc::new(|_: &str, _: String| {})); state.build_now(Some("quick"), Arc::new(|_: &str, _: String| {})); let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); loop { assert!( std::time::Instant::now() < deadline, "quick never finished, so it was waiting on slow", ); let status = state.status(); let quick = status.components.iter().find(|c| c.name == "quick"); if quick.is_some_and(|quick| quick.step.is_none()) { assert!( status .components .iter() .any(|c| c.name == "slow" && c.step.is_some()), "slow finished first, so this proves nothing about waiting", ); break; } tokio::time::sleep(std::time::Duration::from_millis(10)).await; } assert!(root.join("quick/quick-built").exists(), "quick really ran"); assert!( !root.join("slow/slow-built").exists(), "slow is still going, which is the point", ); while state.status().building { assert!( std::time::Instant::now() < deadline, "slow did not finish in time" ); tokio::time::sleep(std::time::Duration::from_millis(10)).await; } assert!( root.join("slow/slow-built").exists(), "and the one that was still going finished on its own", ); } /// A component's failure is its own, and does not become the /// project's. /// /// With one error slot per project, a failed build reported itself on /// every component's card and turned every Update button into Retry. #[tokio::test] async fn a_failure_is_reported_against_the_component_that_produced_it() { let dir = tempfile::tempdir().expect("tempdir"); let root = dir.path(); for name in ["broken", "fine"] { std::fs::create_dir_all(root.join(name)).expect("mkdir"); } let components = ["broken", "fine"] .map(|name| Component::Apk { name: name.to_string(), modes: Vec::new(), build: crate::config::ByMode::One(crate::config::Command::from_line(match name { "broken" => "false", _ => "true", })), cwd: Some(PathBuf::from(name)), stale_when: None, also_watch: Vec::new(), strip: false, enroll: crate::config::Command::default(), strip_here: None, mode: None, package: None, built_from: None, built_mode: None, }) .to_vec(); let state = state_for(root, components); state.build_now(None, Arc::new(|_: &str, _: String| {})); let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); while state.status().building { assert!( std::time::Instant::now() < deadline, "build did not finish in time" ); tokio::time::sleep(std::time::Duration::from_millis(10)).await; } let status = state.status(); assert!( status.error.is_none(), "the project's error slot is for pulls, not for a component's build", ); let component = |name: &str| { status .components .iter() .find(|c| c.name == name) .expect("an entry per component") .error .clone() }; assert!(component("broken").is_some(), "the one that failed says so"); assert!( component("fine").is_none(), "and the one that did not is left alone", ); assert!(state.build_failed("broken")); assert!( !state.build_failed("fine"), "so its card opens the runtime log, not a build log it has no failure in", ); } /// A sibling that has already been built must not mask that this one /// never has. /// /// The bug the previous test's fix uncovered: `APK_PATTERNS` includes /// a one-level pattern (`*/build/outputs/apk/*/*.apk`), so a second /// component's output sitting one directory below the project root /// satisfies a scan of the *root* even when it belongs to a component /// nobody has ever built here. `component_is_stale`'s "never built" /// check used to run against the project root rather than the /// component's own directory, so the moment either of two components /// had been built once, the other quietly stopped being offered its /// own first build -- `prepare` (what Update runs before a download) /// saw a component with a command and no output and declared it /// current anyway. #[tokio::test] async fn a_sibling_already_built_does_not_hide_that_this_one_never_has() { let dir = tempfile::tempdir().expect("tempdir"); let root = dir.path(); std::fs::create_dir_all(root.join("a")).expect("mkdir"); std::fs::create_dir_all(root.join("b")).expect("mkdir"); let components = vec![ Component::Apk { name: "a".to_string(), modes: Vec::new(), build: crate::config::ByMode::One(crate::config::Command::from_line( "touch a-built", )), cwd: Some(PathBuf::from("a")), stale_when: None, also_watch: Vec::new(), strip: false, enroll: crate::config::Command::default(), strip_here: None, mode: None, package: None, built_from: None, built_mode: None, }, Component::Apk { name: "b".to_string(), modes: Vec::new(), build: crate::config::ByMode::One(crate::config::Command::from_line( "touch b-built", )), cwd: Some(PathBuf::from("b")), stale_when: None, also_watch: Vec::new(), strip: false, enroll: crate::config::Command::default(), strip_here: None, mode: None, package: None, built_from: None, built_mode: None, }, ]; let state = state_for(root, components); // `a` has an APK on disk already -- built by something other than // this state, which is the ordinary case for "added after the // fact" -- while `b` has never been built at all. std::fs::create_dir_all(root.join("a/build/outputs/apk/debug")).expect("mkdir"); std::fs::write(root.join("a/build/outputs/apk/debug/a.apk"), b"").expect("write"); assert!( stale(&state, "b"), "b has never been built, regardless of what a has on disk", ); state.trigger_if_needed(Some("b"), Arc::new(|_: &str, _: String| {})); let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); while state.status().building { assert!( std::time::Instant::now() < deadline, "build did not finish in time" ); tokio::time::sleep(std::time::Duration::from_millis(10)).await; } assert!( root.join("b/b-built").exists(), "prepare must have run b's build" ); } fn state_for(root: &Path, components: Vec) -> Arc { BuildState::new( "test".to_string(), components, true, false, root.to_path_buf(), Shared { remote_checks: crate::git::RemoteChecks::default(), downloads: Arc::new(crate::restart::Downloads::default()), }, false, ) } fn stale(state: &BuildState, name: &str) -> bool { let component = state .components .iter() .find(|component| component.name() == name) .expect("component"); state.component_is_stale(component, &state.inner.lock().unwrap().built_from.clone()) } /// The trap a per-component staleness check walks straight into once /// modes exist. An APK's "never built at all" test finds *any* build /// under the component's directory, so a debug APK sitting there is /// enough to make a component switched to release look built -- and /// switching modes moves no commit, so nothing else has an opinion. /// Update would then do nothing and the phone would install the debug /// build from a card saying release. /// /// Asserted through `component_is_stale` rather than through /// `built_in_another_mode` alone, because the rule being right is not /// the same as the check that decides whether to build asking it. #[test] fn a_component_switched_to_another_mode_is_stale_even_with_a_build_on_disk() { let dir = tempfile::tempdir().expect("tempdir"); let root = dir.path(); // A build output where an APK actually lands, so the "nothing // built here" branch is satisfied and cannot be what answers. let built = root.join("build/outputs/apk/debug"); std::fs::create_dir_all(&built).expect("mkdir"); std::fs::write(built.join("app-debug.apk"), b"not really an apk").expect("write"); let moded = |mode: Option<&str>| { vec![Component::Apk { name: "app".to_string(), modes: vec!["release".to_string(), "debug".to_string()], build: crate::config::ByMode::One(crate::config::Command::from_line("true")), cwd: None, stale_when: None, also_watch: Vec::new(), strip: false, enroll: crate::config::Command::default(), strip_here: None, package: None, mode: mode.map(str::to_string), // Built here once, in debug -- without which there is no // build of ours for the comparison to be about. built_from: Some("abc123".to_string()), built_mode: Some("debug".to_string()), }] }; let matching = state_for(root, moded(Some("debug"))); assert!( !stale(&matching, "app"), "the build on disk is this mode's, so there is nothing to do", ); let switched = state_for(root, moded(Some("release"))); assert!( stale(&switched, "app"), "a debug build must not satisfy a component set to build release", ); assert_eq!( switched.freshness(&switched.components[0]), Freshness::Behind, "and the card has to say so rather than reading as current", ); } /// The shape of bug a unit test that only calls `component_is_stale` /// cannot see: `status()` and `trigger_if_needed` hold the lock that /// the staleness check also wants. Taking it twice on one thread is a /// deadlock, and it hung every `/status` and `/prepare` for any /// project that had a build on disk -- which is every real one. /// /// Run on a thread with a deadline so a regression fails in /// milliseconds instead of hanging the suite. #[test] fn asking_for_status_does_not_deadlock() { let dir = tempfile::tempdir().expect("tempdir"); let root = dir.path(); let components = two_component_checkout(root); let state = state_for(root, components); let (tx, rx) = std::sync::mpsc::channel(); std::thread::spawn(move || { let _ = state.status(); let _ = tx.send(()); }); assert!( rx.recv_timeout(std::time::Duration::from_secs(5)).is_ok(), "status() deadlocked on its own lock", ); } /// A commit that touches one component's directory must not make the /// other look behind. Without the scoping, every app-only change would /// offer a pointless rebuild of the server -- in the one repository /// shape this feature was asked for. #[test] fn a_commit_in_one_subtree_leaves_the_other_current() { let dir = tempfile::tempdir().expect("tempdir"); let root = dir.path(); let components = two_component_checkout(root); let state = state_for(root, components); // Built now, from what is checked out now. for name in ["backend", "app"] { let cwd = PathBuf::from(if name == "backend" { "server" } else { "app" }); let sha = crate::git::subtree_head(root, std::slice::from_ref(&cwd)).expect("a commit"); state .inner .lock() .unwrap() .built_from .insert(name.to_string(), sha); } assert!(!stale(&state, "backend"), "just built"); assert!(!stale(&state, "app"), "just built"); // A change to the app only. std::fs::write(root.join("app/main.kt"), "two").expect("write"); run(root, &["git", "commit", "-qam", "app only"]); assert!(stale(&state, "app"), "the app moved and was not rebuilt"); assert!( !stale(&state, "backend"), "the server's directory did not change" ); } /// The other half of that scoping: a component's build reads more /// than its own directory, and what it names with `alsoWatch` counts /// as its own. /// /// The case this is for had every piece defensible and the whole /// silent: two clients sourcing one shared script that locates their /// SDK and mints their signing key, a fix committed to that script, /// and no component's subtree head moved -- so nothing rebuilt, and /// every card correctly reported "current" while answering a /// narrower question than the reader was asking. The component that /// says nothing must still be left alone, which is the second /// assertion here. #[test] fn a_commit_to_a_watched_path_makes_only_the_components_that_watch_it_stale() { let dir = tempfile::tempdir().expect("tempdir"); let root = dir.path(); let mut components = two_component_checkout(root); let Component::Apk { also_watch, .. } = &mut components[1] else { panic!("the app is the second component"); }; *also_watch = vec![PathBuf::from("scripts")]; let state = state_for(root, components); std::fs::create_dir_all(root.join("scripts")).expect("mkdir"); std::fs::write(root.join("scripts/env.sh"), "one").expect("write"); run(root, &["git", "add", "-A"]); run(root, &["git", "commit", "-qm", "scripts"]); for name in ["backend", "app"] { let component = state .components .iter() .find(|component| component.name() == name) .expect("component"); let sha = crate::git::subtree_head(root, &component.watched_paths()).expect("a commit"); state .inner .lock() .unwrap() .built_from .insert(name.to_string(), sha); } assert!(!stale(&state, "backend"), "just built"); assert!(!stale(&state, "app"), "just built"); // A change to neither component's own directory. std::fs::write(root.join("scripts/env.sh"), "two").expect("write"); run(root, &["git", "commit", "-qam", "shared script"]); assert!( stale(&state, "app"), "the app watches scripts/, so a commit there is a commit to it", ); assert!( !stale(&state, "backend"), "the server said nothing about scripts/, so nothing about it changed", ); } /// Nothing recorded is "we have never built this", not "this is out of /// date" -- otherwise every project that existed before this feature /// would demand a rebuild the moment it arrived. #[test] fn a_component_this_server_has_never_built_is_not_called_stale() { let dir = tempfile::tempdir().expect("tempdir"); let root = dir.path(); let components = two_component_checkout(root); let state = state_for(root, components); assert!( state.inner.lock().unwrap().built_from.is_empty(), "nothing recorded" ); assert!(!stale(&state, "backend")); assert!(!stale(&state, "app")); } /// The same must hold where there is no checkout to compare against: /// unreadable is not evidence of anything. #[test] fn a_project_outside_git_is_not_called_stale() { let dir = tempfile::tempdir().expect("tempdir"); let root = dir.path(); std::fs::create_dir_all(root.join("app/build/outputs/apk/debug")).expect("mkdir"); std::fs::write(root.join("app/build/outputs/apk/debug/a.apk"), b"").expect("write"); let state = state_for( root, vec![Component::Apk { name: "app".to_string(), modes: Vec::new(), build: crate::config::ByMode::One(crate::config::Command::from_line("true")), cwd: None, stale_when: None, also_watch: Vec::new(), strip: false, enroll: crate::config::Command::default(), strip_here: None, package: None, mode: None, built_from: Some("0000000000000000000000000000000000000000".to_string()), built_mode: None, }], ); assert!(!stale(&state, "app"), "no checkout means no opinion"); } /// The marker is recognised only in exactly its shape; anything else on /// a line is the build talking and belongs in the log, where a build /// that prints a malformed report can be seen to have done so. #[test] fn progress_is_read_only_from_the_marker() { assert_eq!(parse_progress("@@progress 12/43"), Some((12, 43))); assert_eq!(parse_progress(" @@progress 0/1 "), Some((0, 1))); assert_eq!(parse_progress("@@progress 7 / 9 "), Some((7, 9))); assert_eq!(parse_progress("Building @@progress 1/2"), None); assert_eq!(parse_progress("@@progress soon"), None); assert_eq!(parse_progress("@@progress 1/"), None); assert_eq!(parse_progress("> Task :app:compileDebugKotlin"), None); } /// Feeds `read_segments` one chunk per `read`, recording how many /// segments had been emitted before each one was handed over. struct Chunks { chunks: std::vec::IntoIter<&'static [u8]>, emitted: std::rc::Rc>>, before_each_read: std::rc::Rc>>, } impl std::io::Read for Chunks { fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { self.before_each_read .borrow_mut() .push(self.emitted.borrow().len()); let Some(chunk) = self.chunks.next() else { return Ok(0); }; assert!(chunk.len() <= buffer.len(), "the test's chunks are small"); buffer[..chunk.len()].copy_from_slice(chunk); Ok(chunk.len()) } } /// The defect this splitting exists for, and the half of it a batch /// test cannot see: it is not whether the counter is *parsed* but /// **when** it is delivered. /// /// Cargo ends its counter with a carriage return and no newline, so a /// reader waiting for `\n` holds it until whatever prints the next /// one. Here that is the `Finished` line, which on a real incremental /// build arrives only when the build is over -- so the card had /// nothing to measure for the whole compile, which is exactly what a /// pull produces. The assertion is therefore about the count of /// segments already emitted when the *second* chunk is asked for. #[test] fn a_carriage_return_delivers_its_segment_without_waiting_for_a_newline() { // Byte for byte what `cargo build --release` printed with one // dirty crate, shortened only in the bar and the path. let chunks: Vec<&'static [u8]> = vec![ b" Compiling dev-updater v0.1.0 (/p)\n Building [==> ] 236/237: dev-updater(bin) \r", b" Finished `release` profile [optimized] target(s) in 7.60s\n", ]; let emitted = std::rc::Rc::new(std::cell::RefCell::new(Vec::new())); let before_each_read = std::rc::Rc::new(std::cell::RefCell::new(Vec::new())); let collected = std::rc::Rc::clone(&emitted); read_segments( Chunks { chunks: chunks.into_iter(), emitted: std::rc::Rc::clone(&emitted), before_each_read: std::rc::Rc::clone(&before_each_read), }, |segment| collected.borrow_mut().push(segment.to_string()), ); assert_eq!( before_each_read.borrow().as_slice(), [0, 2, 3], "the counter must be delivered with its own chunk, not held \ until the newline in the next one" ); let emitted = emitted.borrow(); assert_eq!(emitted.len(), 3); assert_eq!( parse_progress(&emitted[1]), Some((236, 237)), "and it is still the cargo counter once split" ); } /// The case the fix was not written for: output that was already /// arriving line by line must not change shape. A cold build is /// mostly this, and it is where a wrapper's `@@progress` lines live /// too. #[test] fn newline_terminated_output_is_unchanged_and_blank_segments_are_dropped() { let mut segments = Vec::new(); read_segments( &b"> Task :app:compileDebugKotlin\n@@progress 7/9\r\n\nlast, unterminated"[..], |segment| segments.push(segment.to_string()), ); assert_eq!( segments, [ "> Task :app:compileDebugKotlin", "@@progress 7/9", "", "", "last, unterminated" ], "a CRLF ends one segment and opens an empty one; the caller \ drops the blanks" ); } }