feat: implement proper task cancellation for scheduled jobs (#2731)

This commit is contained in:
Max Novich
2025-05-29 18:33:27 -07:00
committed by GitHub
parent a05029773d
commit bd430866e8
12 changed files with 2945 additions and 237 deletions
+4
View File
@@ -45,6 +45,8 @@ use utoipa::OpenApi;
super::routes::schedule::run_now_handler,
super::routes::schedule::pause_schedule,
super::routes::schedule::unpause_schedule,
super::routes::schedule::kill_running_job,
super::routes::schedule::inspect_running_job,
super::routes::schedule::sessions_handler
),
components(schemas(
@@ -95,6 +97,8 @@ use utoipa::OpenApi;
SessionMetadata,
super::routes::schedule::CreateScheduleRequest,
super::routes::schedule::UpdateScheduleRequest,
super::routes::schedule::KillJobResponse,
super::routes::schedule::InspectJobResponse,
goose::scheduler::ScheduledJob,
super::routes::schedule::RunNowResponse,
super::routes::schedule::ListSchedulesResponse,
+116
View File
@@ -31,6 +31,21 @@ pub struct ListSchedulesResponse {
jobs: Vec<ScheduledJob>,
}
// Response for the kill endpoint
#[derive(Serialize, utoipa::ToSchema)]
pub struct KillJobResponse {
message: String,
}
// Response for the inspect endpoint
#[derive(Serialize, utoipa::ToSchema)]
#[serde(rename_all = "camelCase")]
pub struct InspectJobResponse {
session_id: Option<String>,
process_start_time: Option<String>,
running_duration_seconds: Option<i64>,
}
// Response for the run_now endpoint
#[derive(Serialize, utoipa::ToSchema)]
pub struct RunNowResponse {
@@ -100,6 +115,8 @@ async fn create_schedule(
last_run: None,
currently_running: false,
paused: false,
current_session_id: None,
process_start_time: None,
};
scheduler
.add_scheduled_job(job.clone())
@@ -199,6 +216,17 @@ async fn run_now_handler(
eprintln!("Error running schedule '{}' now: {:?}", id, e);
match e {
goose::scheduler::SchedulerError::JobNotFound(_) => Err(StatusCode::NOT_FOUND),
goose::scheduler::SchedulerError::AnyhowError(ref err) => {
// Check if this is a cancellation error
if err.to_string().contains("was successfully cancelled") {
// Return a special session_id to indicate cancellation
Ok(Json(RunNowResponse {
session_id: "CANCELLED".to_string(),
}))
} else {
Err(StatusCode::INTERNAL_SERVER_ERROR)
}
}
_ => Err(StatusCode::INTERNAL_SERVER_ERROR),
}
}
@@ -389,6 +417,92 @@ async fn update_schedule(
Ok(Json(updated_job))
}
#[utoipa::path(
post,
path = "/schedule/{id}/kill",
responses(
(status = 200, description = "Running job killed successfully"),
),
tag = "schedule"
)]
#[axum::debug_handler]
pub async fn kill_running_job(
State(state): State<Arc<AppState>>,
headers: HeaderMap,
Path(id): Path<String>,
) -> Result<Json<KillJobResponse>, StatusCode> {
verify_secret_key(&headers, &state)?;
let scheduler = state
.scheduler()
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
scheduler.kill_running_job(&id).await.map_err(|e| {
eprintln!("Error killing running job '{}': {:?}", id, e);
match e {
goose::scheduler::SchedulerError::JobNotFound(_) => StatusCode::NOT_FOUND,
goose::scheduler::SchedulerError::AnyhowError(_) => StatusCode::BAD_REQUEST,
_ => StatusCode::INTERNAL_SERVER_ERROR,
}
})?;
Ok(Json(KillJobResponse {
message: format!("Successfully killed running job '{}'", id),
}))
}
#[utoipa::path(
get,
path = "/schedule/{id}/inspect",
params(
("id" = String, Path, description = "ID of the schedule to inspect")
),
responses(
(status = 200, description = "Running job information", body = InspectJobResponse),
(status = 404, description = "Scheduled job not found"),
(status = 500, description = "Internal server error")
),
tag = "schedule"
)]
#[axum::debug_handler]
pub async fn inspect_running_job(
State(state): State<Arc<AppState>>,
headers: HeaderMap,
Path(id): Path<String>,
) -> Result<Json<InspectJobResponse>, StatusCode> {
verify_secret_key(&headers, &state)?;
let scheduler = state
.scheduler()
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
match scheduler.get_running_job_info(&id).await {
Ok(info) => {
if let Some((session_id, start_time)) = info {
let duration = chrono::Utc::now().signed_duration_since(start_time);
Ok(Json(InspectJobResponse {
session_id: Some(session_id),
process_start_time: Some(start_time.to_rfc3339()),
running_duration_seconds: Some(duration.num_seconds()),
}))
} else {
Ok(Json(InspectJobResponse {
session_id: None,
process_start_time: None,
running_duration_seconds: None,
}))
}
}
Err(e) => {
eprintln!("Error inspecting running job '{}': {:?}", id, e);
match e {
goose::scheduler::SchedulerError::JobNotFound(_) => Err(StatusCode::NOT_FOUND),
_ => Err(StatusCode::INTERNAL_SERVER_ERROR),
}
}
}
}
pub fn routes(state: Arc<AppState>) -> Router {
Router::new()
.route("/schedule/create", post(create_schedule))
@@ -398,6 +512,8 @@ pub fn routes(state: Arc<AppState>) -> Router {
.route("/schedule/{id}/run_now", post(run_now_handler)) // Corrected
.route("/schedule/{id}/pause", post(pause_schedule))
.route("/schedule/{id}/unpause", post(unpause_schedule))
.route("/schedule/{id}/kill", post(kill_running_job))
.route("/schedule/{id}/inspect", get(inspect_running_job))
.route("/schedule/{id}/sessions", get(sessions_handler)) // Corrected
.with_state(state)
}