fix(developer): terminate owned shell process trees on cancellation (#12958)
Signed-off-by: Jack Amadeo <jackamadeo@squareup.com> Co-authored-by: goose <noreply@block.xyz>
This commit is contained in:
@@ -257,9 +257,9 @@ gethostname = "1.1.0"
|
||||
boa_engine = { version = "0.22", default-features = false, optional = true, features = ["fuzz"] }
|
||||
|
||||
[target.'cfg(target_os = "windows")'.dependencies]
|
||||
winapi = { workspace = true, features = ["accctrl", "aclapi", "fileapi", "handleapi", "minwinbase", "sddl", "securitybaseapi", "winbase", "winerror"] }
|
||||
winapi = { workspace = true, features = ["accctrl", "aclapi", "fileapi", "handleapi", "jobapi2", "minwinbase", "processthreadsapi", "sddl", "securitybaseapi", "synchapi", "winbase", "winerror"] }
|
||||
keyring = { workspace = true, features = ["windows-native"], optional = true }
|
||||
ntapi = { version = "0.4.3", default-features = false }
|
||||
ntapi = { version = "0.4.3", default-features = false, features = ["user"] }
|
||||
|
||||
# Platform-specific GPU acceleration for Whisper and local inference
|
||||
[target.'cfg(target_os = "macos")'.dependencies]
|
||||
|
||||
@@ -2,6 +2,7 @@ pub mod edit;
|
||||
pub mod image;
|
||||
pub mod shell;
|
||||
mod shell_output_streaming;
|
||||
mod shell_process;
|
||||
pub mod tree;
|
||||
|
||||
use crate::agents::mcp_client::{Error, McpClientTrait};
|
||||
|
||||
@@ -11,6 +11,7 @@ use serde::{Deserialize, Serialize};
|
||||
use tokio::io::{AsyncBufReadExt, BufReader};
|
||||
use tokio_stream::{wrappers::SplitStream, StreamExt};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tokio_util::task::AbortOnDropHandle;
|
||||
|
||||
use crate::agents::tool_execution::ToolCallNotificationEmitter;
|
||||
use crate::subprocess::SubprocessExt;
|
||||
@@ -20,6 +21,7 @@ pub use super::shell_output_streaming::{
|
||||
ShellOutputStream, DEVELOPER_SHELL_OUTPUT_NOTIFICATION_METHOD,
|
||||
};
|
||||
use super::shell_output_streaming::{ShellOutputBatcher, SHELL_LIVE_OUTPUT_FLUSH_INTERVAL};
|
||||
use super::shell_process::ShellProcess;
|
||||
|
||||
/// Check if the current process is running inside a Flatpak sandbox.
|
||||
///
|
||||
@@ -431,51 +433,53 @@ async fn run_command(
|
||||
};
|
||||
|
||||
let mut command = tokio::select! {
|
||||
command = build_shell_command(command_line, working_dir, session_id) => command,
|
||||
_ = sleep_until_deadline(deadline) => return Ok(not_started(true)),
|
||||
biased;
|
||||
_ = cancellation_token.cancelled() => return Ok(not_started(false)),
|
||||
_ = sleep_until_deadline(deadline) => return Ok(not_started(true)),
|
||||
command = build_shell_command(command_line, working_dir, session_id) => command,
|
||||
};
|
||||
|
||||
command.stdout(Stdio::piped());
|
||||
command.stderr(Stdio::piped());
|
||||
command.stdin(Stdio::null());
|
||||
|
||||
let mut child = command
|
||||
.spawn()
|
||||
let mut process = ShellProcess::spawn(&mut command)
|
||||
.map_err(|error| format!("Failed to spawn shell command: {}", error))?;
|
||||
|
||||
let child_stdout = child
|
||||
let child_stdout = process
|
||||
.child
|
||||
.stdout
|
||||
.take()
|
||||
.ok_or_else(|| "Failed to capture stdout".to_string())?;
|
||||
let child_stderr = child
|
||||
let child_stderr = process
|
||||
.child
|
||||
.stderr
|
||||
.take()
|
||||
.ok_or_else(|| "Failed to capture stderr".to_string())?;
|
||||
|
||||
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
|
||||
let output_task = tokio::spawn(collect_tagged_lines(
|
||||
let output_task = AbortOnDropHandle::new(tokio::spawn(collect_tagged_lines(
|
||||
child_stdout,
|
||||
child_stderr,
|
||||
tx,
|
||||
notification_emitter,
|
||||
));
|
||||
)));
|
||||
let abort_handle = output_task.abort_handle();
|
||||
|
||||
let mut timed_out = false;
|
||||
let exit_code = tokio::select! {
|
||||
result = child.wait() => result
|
||||
result = process.wait() => result
|
||||
.map_err(|error| format!("Failed waiting on shell command: {}", error))?
|
||||
.code(),
|
||||
_ = sleep_until_deadline(deadline) => {
|
||||
timed_out = true;
|
||||
let _ = child.start_kill();
|
||||
let _ = child.wait().await;
|
||||
process.kill().await
|
||||
.map_err(|error| format!("Failed to stop shell command: {}", error))?;
|
||||
None
|
||||
}
|
||||
_ = cancellation_token.cancelled() => {
|
||||
let _ = child.start_kill();
|
||||
let _ = child.wait().await;
|
||||
process.kill().await
|
||||
.map_err(|error| format!("Failed to stop shell command: {}", error))?;
|
||||
None
|
||||
}
|
||||
};
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
use std::io;
|
||||
use std::process::ExitStatus;
|
||||
use tokio::process::{Child, Command};
|
||||
|
||||
pub(super) struct ShellProcess {
|
||||
pub(super) child: Child,
|
||||
tree: platform::ProcessTree,
|
||||
armed: bool,
|
||||
}
|
||||
|
||||
impl ShellProcess {
|
||||
pub(super) fn spawn(command: &mut Command) -> io::Result<Self> {
|
||||
let tree = platform::ProcessTree::new()?;
|
||||
tree.configure(command);
|
||||
command.kill_on_drop(true);
|
||||
let mut process = Self {
|
||||
child: command.spawn()?,
|
||||
tree,
|
||||
armed: true,
|
||||
};
|
||||
process.tree.attach(&process.child)?;
|
||||
Ok(process)
|
||||
}
|
||||
|
||||
pub(super) async fn wait(&mut self) -> io::Result<ExitStatus> {
|
||||
let status = self.child.wait().await?;
|
||||
// Successful shell exit may leave deliberately backgrounded work alive.
|
||||
// On Unix, also stop retaining a PGID as soon as its leader is reaped.
|
||||
self.armed = false;
|
||||
self.tree.disarm()?;
|
||||
Ok(status)
|
||||
}
|
||||
|
||||
pub(super) async fn kill(&mut self) -> io::Result<()> {
|
||||
self.tree.terminate()?;
|
||||
self.armed = false;
|
||||
self.tree.disarm()?;
|
||||
self.child.wait().await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ShellProcess {
|
||||
fn drop(&mut self) {
|
||||
if self.armed {
|
||||
let _ = self.tree.terminate();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
mod platform {
|
||||
use super::*;
|
||||
|
||||
pub(super) struct ProcessTree {
|
||||
pgid: Option<libc::pid_t>,
|
||||
}
|
||||
|
||||
impl ProcessTree {
|
||||
pub(super) fn new() -> io::Result<Self> {
|
||||
Ok(Self { pgid: None })
|
||||
}
|
||||
|
||||
pub(super) fn configure(&self, command: &mut Command) {
|
||||
command.process_group(0);
|
||||
}
|
||||
|
||||
pub(super) fn attach(&mut self, child: &Child) -> io::Result<()> {
|
||||
self.pgid = Some(child.id().expect("freshly spawned shell has a PID") as libc::pid_t);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn terminate(&self) -> io::Result<()> {
|
||||
if let Some(pgid) = self.pgid {
|
||||
// SAFETY: this positive PGID belongs to the fresh process group
|
||||
// created for this invocation, never Goose's shared group.
|
||||
if unsafe { libc::kill(-pgid, libc::SIGKILL) } != 0 {
|
||||
let error = io::Error::last_os_error();
|
||||
if error.raw_os_error() != Some(libc::ESRCH) {
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn disarm(&mut self) -> io::Result<()> {
|
||||
self.pgid = None;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
mod platform {
|
||||
use super::*;
|
||||
use ntapi::ntpsapi::NtResumeProcess;
|
||||
use std::os::windows::io::{AsRawHandle, FromRawHandle, OwnedHandle};
|
||||
use winapi::um::jobapi2::{
|
||||
AssignProcessToJobObject, CreateJobObjectW, SetInformationJobObject, TerminateJobObject,
|
||||
};
|
||||
use winapi::um::winbase::{CREATE_NO_WINDOW, CREATE_SUSPENDED};
|
||||
use winapi::um::winnt::{
|
||||
JobObjectExtendedLimitInformation, JOBOBJECT_EXTENDED_LIMIT_INFORMATION,
|
||||
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE,
|
||||
};
|
||||
|
||||
pub(super) struct ProcessTree {
|
||||
job: OwnedHandle,
|
||||
}
|
||||
|
||||
impl ProcessTree {
|
||||
pub(super) fn new() -> io::Result<Self> {
|
||||
// SAFETY: null attributes/name create an unnamed, noninheritable job.
|
||||
let handle = unsafe { CreateJobObjectW(std::ptr::null_mut(), std::ptr::null()) };
|
||||
if handle.is_null() {
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
let tree = Self {
|
||||
// SAFETY: CreateJobObjectW returned a new handle owned here.
|
||||
job: unsafe { OwnedHandle::from_raw_handle(handle) },
|
||||
};
|
||||
tree.set_kill_on_close(true)?;
|
||||
Ok(tree)
|
||||
}
|
||||
|
||||
pub(super) fn configure(&self, command: &mut Command) {
|
||||
// Assign before executing user code, so even immediately spawned
|
||||
// grandchildren inherit the job rather than escaping assignment.
|
||||
command.creation_flags(CREATE_NO_WINDOW | CREATE_SUSPENDED);
|
||||
}
|
||||
|
||||
pub(super) fn attach(&mut self, child: &Child) -> io::Result<()> {
|
||||
let process = child
|
||||
.raw_handle()
|
||||
.expect("freshly spawned shell has a handle");
|
||||
// SAFETY: both handles are live; the child is still suspended.
|
||||
if unsafe { AssignProcessToJobObject(self.job.as_raw_handle(), process) } == 0 {
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
// Tokio exposes the process handle, not the primary thread handle.
|
||||
// Resume the process only after it belongs to this invocation's job.
|
||||
// SAFETY: the suspended child process handle remains live for the call.
|
||||
let status = unsafe { NtResumeProcess(process) };
|
||||
if status < 0 {
|
||||
return Err(io::Error::other(format!(
|
||||
"Failed to resume shell process: NTSTATUS {status:#x}"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn terminate(&self) -> io::Result<()> {
|
||||
// SAFETY: the handle identifies only this shell invocation's job.
|
||||
if unsafe { TerminateJobObject(self.job.as_raw_handle(), 1) } == 0 {
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn set_kill_on_close(&self, enabled: bool) -> io::Result<()> {
|
||||
// SAFETY: zeroed is a valid initial limit structure, and its address
|
||||
// and exact size are provided only for the duration of this call.
|
||||
let mut limits: JOBOBJECT_EXTENDED_LIMIT_INFORMATION = unsafe { std::mem::zeroed() };
|
||||
if enabled {
|
||||
limits.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
|
||||
}
|
||||
if unsafe {
|
||||
SetInformationJobObject(
|
||||
self.job.as_raw_handle(),
|
||||
JobObjectExtendedLimitInformation,
|
||||
&mut limits as *mut _ as *mut _,
|
||||
std::mem::size_of_val(&limits) as u32,
|
||||
)
|
||||
} == 0
|
||||
{
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn disarm(&mut self) -> io::Result<()> {
|
||||
self.set_kill_on_close(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,386 @@
|
||||
#![cfg(any(unix, windows))]
|
||||
|
||||
use goose::agents::platform_extensions::developer::shell::{ShellOutput, ShellParams, ShellTool};
|
||||
use rmcp::model::CallToolResult;
|
||||
use std::path::Path;
|
||||
use std::process::{Child, Command, Stdio};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tokio_util::task::AbortOnDropHandle;
|
||||
|
||||
const HELPER_ENV: &str = "GOOSE_SHELL_CLEANUP_HELPER";
|
||||
const HELPER_ARG: &str = "--goose-shell-cleanup-helper";
|
||||
const WAIT: Duration = Duration::from_secs(10);
|
||||
const LIFETIME: Duration = Duration::from_secs(30);
|
||||
|
||||
// These exercise native shells, not Flatpak's separate host transport.
|
||||
#[ctor::ctor]
|
||||
unsafe fn run_helper() {
|
||||
if std::env::var_os(HELPER_ENV).is_none()
|
||||
|| std::env::args().nth(1).as_deref() != Some(HELPER_ARG)
|
||||
{
|
||||
return;
|
||||
}
|
||||
let mode = std::env::args().nth(2).expect("helper mode");
|
||||
let deadline = Instant::now() + LIFETIME;
|
||||
if mode == "leaf" {
|
||||
std::fs::write("leaf.pid.tmp", std::process::id().to_string()).unwrap();
|
||||
std::fs::rename("leaf.pid.tmp", "leaf.pid").unwrap();
|
||||
while Instant::now() < deadline {
|
||||
if Path::new("probe").exists() {
|
||||
std::fs::write("leaf.alive", "alive").unwrap();
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
}
|
||||
std::process::exit(0);
|
||||
}
|
||||
|
||||
std::fs::write("parent.pid", std::process::id().to_string()).unwrap();
|
||||
let mut leaf = Command::new(std::env::current_exe().unwrap())
|
||||
.args([HELPER_ARG, "leaf"])
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
while !Path::new("leaf.pid").exists() {
|
||||
if Instant::now() >= deadline || leaf.try_wait().unwrap().is_some() {
|
||||
let _ = leaf.kill();
|
||||
let _ = leaf.wait();
|
||||
std::process::exit(1);
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
}
|
||||
println!("helper stdout ready");
|
||||
eprintln!("helper stderr ready");
|
||||
std::fs::write("tree.ready", "ready").unwrap();
|
||||
while !Path::new("finish").exists() && Instant::now() < deadline {
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
}
|
||||
if mode == "background" {
|
||||
std::process::exit(0);
|
||||
}
|
||||
let _ = leaf.kill();
|
||||
let _ = leaf.wait();
|
||||
std::process::exit(0);
|
||||
}
|
||||
|
||||
struct Process {
|
||||
pid: u32,
|
||||
#[cfg(windows)]
|
||||
handle: std::os::windows::io::OwnedHandle,
|
||||
}
|
||||
|
||||
impl Process {
|
||||
fn retain(pid: u32) -> Option<Self> {
|
||||
#[cfg(windows)]
|
||||
{
|
||||
use std::os::windows::io::FromRawHandle;
|
||||
use winapi::um::processthreadsapi::OpenProcess;
|
||||
use winapi::um::winnt::{PROCESS_TERMINATE, SYNCHRONIZE};
|
||||
let handle = unsafe { OpenProcess(PROCESS_TERMINATE | SYNCHRONIZE, 0, pid) };
|
||||
if handle.is_null() {
|
||||
return None;
|
||||
}
|
||||
Some(Self {
|
||||
pid,
|
||||
handle: unsafe { std::os::windows::io::OwnedHandle::from_raw_handle(handle) },
|
||||
})
|
||||
}
|
||||
#[cfg(unix)]
|
||||
Some(Self { pid })
|
||||
}
|
||||
|
||||
fn running(&self) -> bool {
|
||||
#[cfg(windows)]
|
||||
{
|
||||
use std::os::windows::io::AsRawHandle;
|
||||
use winapi::shared::winerror::WAIT_TIMEOUT;
|
||||
use winapi::um::synchapi::WaitForSingleObject;
|
||||
use winapi::um::winbase::WAIT_OBJECT_0;
|
||||
let status = unsafe { WaitForSingleObject(self.handle.as_raw_handle(), 0) };
|
||||
assert!(
|
||||
status == WAIT_OBJECT_0 || status == WAIT_TIMEOUT,
|
||||
"process wait failed"
|
||||
);
|
||||
status == WAIT_TIMEOUT
|
||||
}
|
||||
#[cfg(unix)]
|
||||
{
|
||||
if unsafe { libc::kill(self.pid as libc::pid_t, 0) } != 0 {
|
||||
assert_eq!(
|
||||
std::io::Error::last_os_error().raw_os_error(),
|
||||
Some(libc::ESRCH)
|
||||
);
|
||||
return false;
|
||||
}
|
||||
// Orphaned grandchildren can remain zombies until the system reaps them.
|
||||
#[cfg(target_os = "linux")]
|
||||
let state = std::fs::read_to_string(format!("/proc/{}/stat", self.pid))
|
||||
.ok()
|
||||
.and_then(|stat| {
|
||||
stat.rsplit_once(") ")
|
||||
.and_then(|(_, tail)| tail.chars().next())
|
||||
});
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let state = {
|
||||
let output = Command::new("ps")
|
||||
.args(["-o", "stat=", "-p", &self.pid.to_string()])
|
||||
.output()
|
||||
.expect("read native process status");
|
||||
String::from_utf8_lossy(&output.stdout)
|
||||
.trim()
|
||||
.chars()
|
||||
.next()
|
||||
};
|
||||
!matches!(state, None | Some('Z' | 'X'))
|
||||
}
|
||||
}
|
||||
|
||||
fn kill(&self) {
|
||||
#[cfg(windows)]
|
||||
{
|
||||
use std::os::windows::io::AsRawHandle;
|
||||
unsafe {
|
||||
winapi::um::processthreadsapi::TerminateProcess(self.handle.as_raw_handle(), 1);
|
||||
}
|
||||
}
|
||||
#[cfg(unix)]
|
||||
if self.running() {
|
||||
unsafe {
|
||||
libc::kill(self.pid as libc::pid_t, libc::SIGKILL);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct Tree {
|
||||
dir: tempfile::TempDir,
|
||||
processes: Vec<Process>,
|
||||
direct_child: Option<Child>,
|
||||
}
|
||||
|
||||
impl Tree {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
dir: tempfile::tempdir().unwrap(),
|
||||
processes: Vec::new(),
|
||||
direct_child: None,
|
||||
}
|
||||
}
|
||||
|
||||
async fn ready(&mut self) {
|
||||
let deadline = Instant::now() + WAIT;
|
||||
while !self.dir.path().join("tree.ready").exists() {
|
||||
assert!(
|
||||
Instant::now() < deadline,
|
||||
"helper tree did not become ready"
|
||||
);
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
for name in ["parent.pid", "leaf.pid"] {
|
||||
let pid = std::fs::read_to_string(self.dir.path().join(name))
|
||||
.unwrap()
|
||||
.parse()
|
||||
.unwrap();
|
||||
self.processes
|
||||
.push(Process::retain(pid).expect("retain live helper process"));
|
||||
}
|
||||
self.assert_running();
|
||||
}
|
||||
|
||||
fn assert_running(&self) {
|
||||
for process in &self.processes {
|
||||
assert!(
|
||||
process.running(),
|
||||
"process {} exited unexpectedly",
|
||||
process.pid
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn stopped(&self) {
|
||||
let deadline = Instant::now() + WAIT;
|
||||
while self.processes.iter().any(Process::running) {
|
||||
assert!(
|
||||
Instant::now() < deadline,
|
||||
"command descendants survived termination"
|
||||
);
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Tree {
|
||||
fn drop(&mut self) {
|
||||
for process in &self.processes {
|
||||
process.kill();
|
||||
}
|
||||
// Read any PIDs not retained yet if a readiness assertion failed.
|
||||
for name in ["leaf.pid", "parent.pid"] {
|
||||
let pid = std::fs::read_to_string(self.dir.path().join(name))
|
||||
.ok()
|
||||
.and_then(|text| text.parse::<u32>().ok());
|
||||
if let Some(pid) = pid.filter(|pid| !self.processes.iter().any(|p| p.pid == *pid)) {
|
||||
if let Some(process) = Process::retain(pid) {
|
||||
process.kill();
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(child) = &mut self.direct_child {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn shell_env() -> env_lock::EnvGuard<'static> {
|
||||
#[cfg(unix)]
|
||||
let shell = "sh";
|
||||
#[cfg(windows)]
|
||||
let shell = "cmd";
|
||||
env_lock::lock_env([("GOOSE_SHELL", Some(shell)), (HELPER_ENV, Some("1"))])
|
||||
}
|
||||
|
||||
fn invoke(
|
||||
tool: &Arc<ShellTool>,
|
||||
tree: &Tree,
|
||||
mode: &str,
|
||||
timeout: u64,
|
||||
token: CancellationToken,
|
||||
) -> AbortOnDropHandle<CallToolResult> {
|
||||
let exe = std::env::current_exe().unwrap();
|
||||
#[cfg(unix)]
|
||||
let command = format!(
|
||||
"'{}' {HELPER_ARG} {mode}; :",
|
||||
exe.to_string_lossy().replace('\'', "'\\''")
|
||||
);
|
||||
#[cfg(windows)]
|
||||
let command = format!("\"\"{}\" {HELPER_ARG} {mode}\"", exe.display());
|
||||
let cwd = tree.dir.path().to_path_buf();
|
||||
let tool = Arc::clone(tool);
|
||||
AbortOnDropHandle::new(tokio::spawn(async move {
|
||||
tool.shell_with_cwd(
|
||||
ShellParams {
|
||||
command,
|
||||
timeout_secs: Some(timeout),
|
||||
},
|
||||
Some(&cwd),
|
||||
None,
|
||||
token,
|
||||
)
|
||||
.await
|
||||
}))
|
||||
}
|
||||
|
||||
async fn output(task: &mut AbortOnDropHandle<CallToolResult>) -> ShellOutput {
|
||||
let result = tokio::time::timeout(WAIT, task)
|
||||
.await
|
||||
.expect("shell did not return promptly")
|
||||
.unwrap();
|
||||
serde_json::from_value(result.structured_content.expect("structured shell output")).unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancellation_kills_tree_without_affecting_concurrent_or_unrelated_processes() {
|
||||
let _env = shell_env();
|
||||
let tool = Arc::new(ShellTool::new().unwrap());
|
||||
let mut cancelled = Tree::new();
|
||||
let mut concurrent = Tree::new();
|
||||
let mut unrelated = Tree::new();
|
||||
unrelated.direct_child = Some(
|
||||
Command::new(std::env::current_exe().unwrap())
|
||||
.args([HELPER_ARG, "tree"])
|
||||
.current_dir(unrelated.dir.path())
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.spawn()
|
||||
.unwrap(),
|
||||
);
|
||||
let token = CancellationToken::new();
|
||||
let mut first = invoke(&tool, &cancelled, "tree", 0, token.clone());
|
||||
let mut second = invoke(&tool, &concurrent, "tree", 0, CancellationToken::new());
|
||||
cancelled.ready().await;
|
||||
concurrent.ready().await;
|
||||
unrelated.ready().await;
|
||||
|
||||
token.cancel();
|
||||
let result = output(&mut first).await;
|
||||
assert_eq!(result.exit_code, None);
|
||||
assert!(!result.timed_out);
|
||||
cancelled.stopped().await;
|
||||
concurrent.assert_running();
|
||||
unrelated.assert_running();
|
||||
assert!(
|
||||
!second.is_finished(),
|
||||
"cancellation ended another invocation"
|
||||
);
|
||||
std::fs::write(concurrent.dir.path().join("finish"), "finish").unwrap();
|
||||
assert_eq!(output(&mut second).await.exit_code, Some(0));
|
||||
concurrent.stopped().await;
|
||||
unrelated.assert_running();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn timeout_kills_tree_and_preserves_output_metadata() {
|
||||
let _env = shell_env();
|
||||
let tool = Arc::new(ShellTool::new().unwrap());
|
||||
let mut tree = Tree::new();
|
||||
let mut task = invoke(&tool, &tree, "tree", 5, CancellationToken::new());
|
||||
tree.ready().await;
|
||||
let result = output(&mut task).await;
|
||||
assert!(result.timed_out);
|
||||
assert_eq!(result.exit_code, None);
|
||||
assert!(result.stdout.contains("helper stdout ready"));
|
||||
assert!(result.stderr.contains("helper stderr ready"));
|
||||
assert!(!result.output_truncated);
|
||||
assert!(result.output_collection_error.is_none());
|
||||
tree.stopped().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn aborting_shell_future_kills_tree() {
|
||||
let _env = shell_env();
|
||||
let tool = Arc::new(ShellTool::new().unwrap());
|
||||
let mut tree = Tree::new();
|
||||
let mut task = invoke(&tool, &tree, "tree", 0, CancellationToken::new());
|
||||
tree.ready().await;
|
||||
task.abort();
|
||||
let error = tokio::time::timeout(WAIT, &mut task)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap_err();
|
||||
assert!(error.is_cancelled());
|
||||
tree.stopped().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn successful_shell_exit_preserves_background_descendant() {
|
||||
let _env = shell_env();
|
||||
let tool = Arc::new(ShellTool::new().unwrap());
|
||||
let mut tree = Tree::new();
|
||||
let mut task = invoke(&tool, &tree, "background", 0, CancellationToken::new());
|
||||
tree.ready().await;
|
||||
std::fs::write(tree.dir.path().join("finish"), "finish").unwrap();
|
||||
let result = output(&mut task).await;
|
||||
assert_eq!(result.exit_code, Some(0));
|
||||
assert!(!result.timed_out);
|
||||
assert!(!result.output_truncated);
|
||||
assert!(result.output_collection_error.is_none());
|
||||
std::fs::write(tree.dir.path().join("probe"), "probe").unwrap();
|
||||
let deadline = Instant::now() + WAIT;
|
||||
while !tree.dir.path().join("leaf.alive").exists() {
|
||||
assert!(
|
||||
tree.processes[1].running(),
|
||||
"successful shell exit killed background work"
|
||||
);
|
||||
assert!(
|
||||
Instant::now() < deadline,
|
||||
"background descendant stopped responding"
|
||||
);
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
assert!(tree.processes[1].running());
|
||||
}
|
||||
Reference in New Issue
Block a user