From 7d8ff8f2c66e113bf3a25c39aa22472c58a75399 Mon Sep 17 00:00:00 2001 From: David Calavera <1050+calavera@users.noreply.github.com> Date: Sat, 1 Nov 2025 13:33:32 -0700 Subject: [PATCH 1/2] Ensure that Oom failure reason is propagated correctly. --- server/src/data_model/mod.rs | 5 +++- server/src/http_objects_v1.rs | 2 ++ server/src/integration_test.rs | 54 ++++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/server/src/data_model/mod.rs b/server/src/data_model/mod.rs index 558e7e0c6..412a4620b 100644 --- a/server/src/data_model/mod.rs +++ b/server/src/data_model/mod.rs @@ -818,6 +818,8 @@ pub enum RequestFailureReason { ConstraintUnsatisfiable, // Cancelled. Cancelled, + // Out of memory. + OutOfMemory, } impl Display for RequestFailureReason { @@ -829,6 +831,7 @@ impl Display for RequestFailureReason { RequestFailureReason::RequestError => "RequestError", RequestFailureReason::ConstraintUnsatisfiable => "ConstraintUnsatisfiable", RequestFailureReason::Cancelled => "Cancelled", + RequestFailureReason::OutOfMemory => "OutOfMemory", }; write!(f, "{str_val}") } @@ -854,7 +857,7 @@ impl From for RequestFailureReason { FunctionRunFailureReason::ConstraintUnsatisfiable => { RequestFailureReason::ConstraintUnsatisfiable } - FunctionRunFailureReason::OutOfMemory => RequestFailureReason::FunctionError, + FunctionRunFailureReason::OutOfMemory => RequestFailureReason::OutOfMemory, } } } diff --git a/server/src/http_objects_v1.rs b/server/src/http_objects_v1.rs index 8089ebc37..83f755b60 100644 --- a/server/src/http_objects_v1.rs +++ b/server/src/http_objects_v1.rs @@ -265,6 +265,7 @@ pub enum RequestFailureReason { RequestError, ConstraintUnsatisfiable, Cancelled, + OutOfMemory, } impl From for RequestFailureReason { @@ -278,6 +279,7 @@ impl From for RequestFailureReason { RequestFailureReason::ConstraintUnsatisfiable } data_model::RequestFailureReason::Cancelled => RequestFailureReason::Cancelled, + data_model::RequestFailureReason::OutOfMemory => RequestFailureReason::OutOfMemory, } } } diff --git a/server/src/integration_test.rs b/server/src/integration_test.rs index cae2013a9..6df136361 100644 --- a/server/src/integration_test.rs +++ b/server/src/integration_test.rs @@ -14,6 +14,8 @@ mod tests { ApplicationState, FunctionRunFailureReason, FunctionRunOutcome, + RequestFailureReason, + RequestOutcome, test_objects::tests::{ TEST_EXECUTOR_ID, TEST_NAMESPACE, @@ -1086,4 +1088,56 @@ mod tests { assert_eq!("2", versions.iter().next().unwrap()); Ok(()) } + + #[tokio::test] + async fn test_request_failure_reason_out_of_memory() -> Result<()> { + let test_srv = testing::TestService::new().await?; + let Service { indexify_state, .. } = test_srv.service.clone(); + + // Invoke the app + let request_id = test_state_store::with_simple_application(&indexify_state).await; + test_srv.process_all_state_changes().await?; + + // register executor + let executor = test_srv + .create_executor(mock_executor_metadata(TEST_EXECUTOR_ID.into())) + .await?; + test_srv.process_all_state_changes().await?; + + // finalize the starting node task with OutOfMemory failure + { + let desired_state = executor.desired_state().await; + assert_eq!(desired_state.allocations.len(), 1); + let allocation = desired_state.allocations.first().unwrap(); + executor + .finalize_allocation( + allocation, + FinalizeFunctionRunArgs::new( + allocation_key_from_proto(allocation), + Some(mock_updates()), + None, + ) + .function_run_outcome(FunctionRunOutcome::Failure( + FunctionRunFailureReason::OutOfMemory, + )), + ) + .await?; + test_srv.process_all_state_changes().await?; + } + + // check that the request outcome is Failure(OutOfMemory) + { + let request_ctx = indexify_state + .reader() + .request_ctx(TEST_NAMESPACE, "graph_A", &request_id)? + .unwrap(); + + assert_eq!( + request_ctx.outcome, + Some(RequestOutcome::Failure(RequestFailureReason::OutOfMemory)) + ); + } + + Ok(()) + } } From 5c36078da3f546288ccaf3b73acd430c8c7adc6b Mon Sep 17 00:00:00 2001 From: David Calavera <1050+calavera@users.noreply.github.com> Date: Sat, 1 Nov 2025 20:13:50 -0700 Subject: [PATCH 2/2] Check that function run and allocation are set as OutOfMemory. --- server/src/integration_test.rs | 79 ++++++++++++++++++++++------------ 1 file changed, 51 insertions(+), 28 deletions(-) diff --git a/server/src/integration_test.rs b/server/src/integration_test.rs index 6df136361..97395d2a9 100644 --- a/server/src/integration_test.rs +++ b/server/src/integration_test.rs @@ -1105,38 +1105,61 @@ mod tests { test_srv.process_all_state_changes().await?; // finalize the starting node task with OutOfMemory failure - { - let desired_state = executor.desired_state().await; - assert_eq!(desired_state.allocations.len(), 1); - let allocation = desired_state.allocations.first().unwrap(); - executor - .finalize_allocation( - allocation, - FinalizeFunctionRunArgs::new( - allocation_key_from_proto(allocation), - Some(mock_updates()), - None, - ) - .function_run_outcome(FunctionRunOutcome::Failure( - FunctionRunFailureReason::OutOfMemory, - )), + let desired_state = executor.desired_state().await; + assert_eq!(desired_state.allocations.len(), 1); + let allocation = desired_state.allocations.first().unwrap(); + executor + .finalize_allocation( + allocation, + FinalizeFunctionRunArgs::new( + allocation_key_from_proto(allocation), + Some(mock_updates()), + None, ) - .await?; - test_srv.process_all_state_changes().await?; - } + .function_run_outcome(FunctionRunOutcome::Failure( + FunctionRunFailureReason::OutOfMemory, + )), + ) + .await?; + test_srv.process_all_state_changes().await?; // check that the request outcome is Failure(OutOfMemory) - { - let request_ctx = indexify_state - .reader() - .request_ctx(TEST_NAMESPACE, "graph_A", &request_id)? - .unwrap(); + let request_ctx = indexify_state + .reader() + .request_ctx(TEST_NAMESPACE, "graph_A", &request_id)? + .unwrap(); - assert_eq!( - request_ctx.outcome, - Some(RequestOutcome::Failure(RequestFailureReason::OutOfMemory)) - ); - } + assert_eq!( + request_ctx.outcome, + Some(RequestOutcome::Failure(RequestFailureReason::OutOfMemory)) + ); + + // check that function_runs have the same failure reason + let function_runs = request_ctx + .function_runs + .values() + .cloned() + .collect::>(); + assert_eq!(function_runs.len(), 1); + let function_run = &function_runs[0]; + assert_eq!( + function_run.outcome, + Some(FunctionRunOutcome::Failure( + FunctionRunFailureReason::OutOfMemory + )) + ); + + // check that allocations have the same failure reason + let allocations = indexify_state + .reader() + .get_allocations_by_request_id(TEST_NAMESPACE, "graph_A", &request_id) + .unwrap(); + assert_eq!(allocations.len(), 1); + let allocation = &allocations[0]; + assert_eq!( + allocation.outcome, + FunctionRunOutcome::Failure(FunctionRunFailureReason::OutOfMemory) + ); Ok(()) }