From 59ac47ba5f883fadee429e9ac0538264fe1a29b8 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Tue, 28 Oct 2025 16:27:12 -0400 Subject: [PATCH 01/29] bump core to 9e9a46191656fc9ccd95589dac3552410561d620 --- packages/core-bridge/Cargo.lock | 5 +++-- packages/core-bridge/sdk-core | 2 +- packages/core-bridge/src/runtime.rs | 12 +++++++++--- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/packages/core-bridge/Cargo.lock b/packages/core-bridge/Cargo.lock index a3f0f9339..eb4c8d686 100644 --- a/packages/core-bridge/Cargo.lock +++ b/packages/core-bridge/Cargo.lock @@ -2509,6 +2509,7 @@ dependencies = [ "tracing", "tracing-core", "url", + "uuid", ] [[package]] @@ -2925,9 +2926,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.18.0" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f33196643e165781c20a5ead5582283a7dacbb87855d867fbc2df3f81eddc1be" +checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" dependencies = [ "getrandom 0.3.3", "js-sys", diff --git a/packages/core-bridge/sdk-core b/packages/core-bridge/sdk-core index de674173c..9e9a46191 160000 --- a/packages/core-bridge/sdk-core +++ b/packages/core-bridge/sdk-core @@ -1 +1 @@ -Subproject commit de674173c664d42f85d0dee1ff3b2ac47e36d545 +Subproject commit 9e9a46191656fc9ccd95589dac3552410561d620 diff --git a/packages/core-bridge/src/runtime.rs b/packages/core-bridge/src/runtime.rs index 42ad067ef..1c6a8e329 100644 --- a/packages/core-bridge/src/runtime.rs +++ b/packages/core-bridge/src/runtime.rs @@ -6,7 +6,7 @@ use neon::prelude::*; use tracing::{Instrument, warn}; use temporal_sdk_core::{ - CoreRuntime, TokioRuntimeBuilder, + CoreRuntime, RuntimeOptionsBuilder, TokioRuntimeBuilder, api::telemetry::{ CoreLog, OtelCollectorOptions as CoreOtelCollectorOptions, PrometheusExporterOptions as CorePrometheusExporterOptions, metrics::CoreMeter, @@ -62,8 +62,14 @@ pub fn runtime_new( let (telemetry_options, metrics_options, logging_options) = bridge_options.try_into()?; // Create core runtime which starts tokio multi-thread runtime - let mut core_runtime = CoreRuntime::new(telemetry_options, TokioRuntimeBuilder::default()) - .context("Failed to initialize Core Runtime")?; + let mut core_runtime = CoreRuntime::new( + RuntimeOptionsBuilder::default() + .telemetry_options(telemetry_options) + .build() + .unwrap(), + TokioRuntimeBuilder::default(), + ) + .context("Failed to initialize Core Runtime")?; enter_sync!(core_runtime); From 7f17d207e078c6f411181406560aac00511f611c Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Wed, 29 Oct 2025 06:47:05 -0400 Subject: [PATCH 02/29] fix tests --- packages/core-bridge/Cargo.lock | 148 +++++++++++------- packages/core-bridge/Cargo.toml | 22 +-- packages/core-bridge/src/worker.rs | 1 + .../test/src/test-worker-poller-autoscale.ts | 2 +- 4 files changed, 106 insertions(+), 67 deletions(-) diff --git a/packages/core-bridge/Cargo.lock b/packages/core-bridge/Cargo.lock index eb4c8d686..c3e0f6f6f 100644 --- a/packages/core-bridge/Cargo.lock +++ b/packages/core-bridge/Cargo.lock @@ -1375,23 +1375,9 @@ checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "opentelemetry" -version = "0.29.1" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e87237e2775f74896f9ad219d26a2081751187eb7c9f5c58dde20a23b95d16c" -dependencies = [ - "futures-core", - "futures-sink", - "js-sys", - "pin-project-lite", - "thiserror 2.0.14", - "tracing", -] - -[[package]] -name = "opentelemetry" -version = "0.30.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf416e4cb72756655126f7dd7bb0af49c674f4c1b9903e80c009e0c37e552e6" +checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0" dependencies = [ "futures-core", "futures-sink", @@ -1403,25 +1389,25 @@ dependencies = [ [[package]] name = "opentelemetry-http" -version = "0.30.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f6639e842a97dbea8886e3439710ae463120091e2e064518ba8e716e6ac36d" +checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" dependencies = [ "async-trait", "bytes", "http", - "opentelemetry 0.30.0", + "opentelemetry", "reqwest", ] [[package]] name = "opentelemetry-otlp" -version = "0.30.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbee664a43e07615731afc539ca60c6d9f1a9425e25ca09c57bc36c87c55852b" +checksum = "7a2366db2dca4d2ad033cad11e6ee42844fd727007af5ad04a1730f4cb8163bf" dependencies = [ "http", - "opentelemetry 0.30.0", + "opentelemetry", "opentelemetry-http", "opentelemetry-proto", "opentelemetry_sdk", @@ -1435,29 +1421,29 @@ dependencies = [ [[package]] name = "opentelemetry-proto" -version = "0.30.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e046fd7660710fe5a05e8748e70d9058dc15c94ba914e7c4faa7c728f0e8ddc" +checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" dependencies = [ - "opentelemetry 0.30.0", + "opentelemetry", "opentelemetry_sdk", "prost", "tonic", + "tonic-prost", ] [[package]] name = "opentelemetry_sdk" -version = "0.30.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11f644aa9e5e31d11896e024305d7e3c98a88884d9f8919dbf37a9991bc47a4b" +checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd" dependencies = [ "futures-channel", "futures-executor", "futures-util", - "opentelemetry 0.30.0", + "opentelemetry", "percent-encoding", "rand 0.9.2", - "serde_json", "thiserror 2.0.14", "tokio", "tokio-stream", @@ -1654,9 +1640,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.13.5" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +checksum = "7231bd9b3d3d33c86b58adbac74b5ec0ad9f496b19d22801d773636feaa95f3d" dependencies = [ "bytes", "prost-derive", @@ -1664,9 +1650,9 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.13.5" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +checksum = "ac6c3320f9abac597dcbc668774ef006702672474aad53c6d596b62e487b40b1" dependencies = [ "heck", "itertools", @@ -1677,6 +1663,8 @@ dependencies = [ "prettyplease", "prost", "prost-types", + "pulldown-cmark", + "pulldown-cmark-to-cmark", "regex", "syn", "tempfile", @@ -1684,9 +1672,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.13.5" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" dependencies = [ "anyhow", "itertools", @@ -1697,18 +1685,18 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.13.5" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +checksum = "b9b4db3d6da204ed77bb26ba83b6122a73aeb2e87e25fbf7ad2e84c4ccbf8f72" dependencies = [ "prost", ] [[package]] name = "prost-wkt" -version = "0.6.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497e1e938f0c09ef9cabe1d49437b4016e03e8f82fbbe5d1c62a9b61b9decae1" +checksum = "655944d0ce015e71b3ec21279437e6a09e58433e50c7b0677901f3d5235e74f5" dependencies = [ "chrono", "inventory", @@ -1721,9 +1709,9 @@ dependencies = [ [[package]] name = "prost-wkt-build" -version = "0.6.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07b8bf115b70a7aa5af1fd5d6e9418492e9ccb6e4785e858c938e28d132a884b" +checksum = "f869f1443fee474b785e935d92e1007f57443e485f51668ed41943fc01a321a2" dependencies = [ "heck", "prost", @@ -1734,9 +1722,9 @@ dependencies = [ [[package]] name = "prost-wkt-types" -version = "0.6.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8cdde6df0a98311c839392ca2f2f0bcecd545f86a62b4e3c6a49c336e970fe5" +checksum = "eeeffd6b9becd4600dd461399f3f71aeda2ff0848802a9ed526cf12e8f42902a" dependencies = [ "chrono", "prost", @@ -1770,6 +1758,26 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "pulldown-cmark" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e8bbe1a966bd2f362681a44f6edce3c2310ac21e4d5067a6e7ec396297a6ea0" +dependencies = [ + "bitflags", + "memchr", + "unicase", +] + +[[package]] +name = "pulldown-cmark-to-cmark" +version = "21.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5b6a0769a491a08b31ea5c62494a8f144ee0987d86d670a8af4df1e1b7cde75" +dependencies = [ + "pulldown-cmark", +] + [[package]] name = "quanta" version = "0.12.6" @@ -2459,7 +2467,7 @@ dependencies = [ "itertools", "lru", "mockall", - "opentelemetry 0.30.0", + "opentelemetry", "opentelemetry-otlp", "opentelemetry_sdk", "parking_lot", @@ -2500,7 +2508,7 @@ dependencies = [ "async-trait", "derive_builder", "derive_more", - "opentelemetry 0.30.0", + "opentelemetry", "prost", "serde_json", "temporal-sdk-core-protos", @@ -2520,16 +2528,15 @@ dependencies = [ "base64", "derive_more", "prost", - "prost-build", "prost-wkt", - "prost-wkt-build", "prost-wkt-types", "rand 0.9.2", "serde", "serde_json", "thiserror 2.0.14", "tonic", - "tonic-build", + "tonic-prost", + "tonic-prost-build", "uuid", ] @@ -2542,7 +2549,7 @@ dependencies = [ "bridge-macros", "futures", "neon", - "opentelemetry 0.29.1", + "opentelemetry", "os_pipe", "parking_lot", "prost", @@ -2706,9 +2713,9 @@ dependencies = [ [[package]] name = "tonic" -version = "0.13.1" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" +checksum = "eb7613188ce9f7df5bfe185db26c5814347d110db17920415cf2fbcad85e7203" dependencies = [ "async-trait", "axum", @@ -2723,9 +2730,9 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "prost", "rustls-native-certs", - "socket2 0.5.10", + "socket2 0.6.0", + "sync_wrapper", "tokio", "tokio-rustls", "tokio-stream", @@ -2737,9 +2744,32 @@ dependencies = [ [[package]] name = "tonic-build" -version = "0.13.1" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac6f67be712d12f0b41328db3137e0d0757645d8904b4cb7d51cd9c2279e847" +checksum = "4c40aaccc9f9eccf2cd82ebc111adc13030d23e887244bc9cfa5d1d636049de3" +dependencies = [ + "prettyplease", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tonic-prost" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66bd50ad6ce1252d87ef024b3d64fe4c3cf54a86fb9ef4c631fdd0ded7aeaa67" +dependencies = [ + "bytes", + "prost", + "tonic", +] + +[[package]] +name = "tonic-prost-build" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4a16cba4043dc3ff43fcb3f96b4c5c154c64cbd18ca8dce2ab2c6a451d058a2" dependencies = [ "prettyplease", "proc-macro2", @@ -2747,6 +2777,8 @@ dependencies = [ "prost-types", "quote", "syn", + "tempfile", + "tonic-build", ] [[package]] @@ -2883,6 +2915,12 @@ dependencies = [ "syn", ] +[[package]] +name = "unicase" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" + [[package]] name = "unicode-ident" version = "1.0.18" diff --git a/packages/core-bridge/Cargo.toml b/packages/core-bridge/Cargo.toml index 1831c7b59..ecc4e78e0 100644 --- a/packages/core-bridge/Cargo.toml +++ b/packages/core-bridge/Cargo.toml @@ -25,28 +25,28 @@ async-trait = "0.1.83" bridge-macros = { path = "bridge-macros" } futures = { version = "0.3", features = ["executor"] } neon = { version = "1.0.0", default-features = false, features = [ - "napi-6", - "futures", + "napi-6", + "futures", ] } -opentelemetry = "0.29" +opentelemetry = "0.31" os_pipe = "1.2.1" parking_lot = "0.12" -prost = "0.13" -prost-types = "0.13" +prost = "0.14" +prost-types = "0.14" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" temporal-sdk-core = { version = "*", path = "./sdk-core/core", features = [ - "ephemeral-server", + "ephemeral-server", ] } temporal-client = { version = "*", path = "./sdk-core/client" } thiserror = "2" tokio = "1.13" tokio-stream = "0.1" -tonic = "0.13" +tonic = "0.14" tracing = "0.1" tracing-subscriber = { version = "0.3", default-features = false, features = [ - "parking_lot", - "env-filter", - "registry", - "ansi", + "parking_lot", + "env-filter", + "registry", + "ansi", ] } diff --git a/packages/core-bridge/src/worker.rs b/packages/core-bridge/src/worker.rs index ca6131f92..a456b92a7 100644 --- a/packages/core-bridge/src/worker.rs +++ b/packages/core-bridge/src/worker.rs @@ -575,6 +575,7 @@ mod config { .max_task_queue_activities_per_second(self.max_task_queue_activities_per_second) .max_worker_activities_per_second(self.max_activities_per_second) .graceful_shutdown_period(self.shutdown_grace_time) + .skip_client_worker_set_check(true) .build() } } diff --git a/packages/test/src/test-worker-poller-autoscale.ts b/packages/test/src/test-worker-poller-autoscale.ts index edd96c8b9..669255066 100644 --- a/packages/test/src/test-worker-poller-autoscale.ts +++ b/packages/test/src/test-worker-poller-autoscale.ts @@ -48,7 +48,7 @@ test.serial('Can run autoscaling polling worker', async (t) => { const activity_pollers = matches.filter((l) => l.includes('activity_task')); t.is(activity_pollers.length, 1, 'Should have exactly one activity poller metric'); t.true(activity_pollers[0].endsWith('2'), 'Activity poller count should be 2'); - const workflow_pollers = matches.filter((l) => l.includes('workflow_task')); + const workflow_pollers = matches.filter((l) => l.includes('workflow_task') && l.includes(taskQueue)); t.is(workflow_pollers.length, 2, 'Should have exactly two workflow poller metrics (sticky and non-sticky)'); // There's sticky & non-sticky pollers, and they may have a count of 1 or 2 depending on From cb50896676ae845efb76f01a6ccb385331a9e89a Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Tue, 28 Oct 2025 16:06:45 -0400 Subject: [PATCH 03/29] add new rpc calls --- packages/core-bridge/src/client.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/core-bridge/src/client.rs b/packages/core-bridge/src/client.rs index 5791328cd..b9bbd405f 100644 --- a/packages/core-bridge/src/client.rs +++ b/packages/core-bridge/src/client.rs @@ -288,6 +288,9 @@ async fn client_invoke_workflow_service( "DescribeDeployment" => { rpc_call!(retry_client, call, describe_deployment) } + "DescribeWorker" => { + rpc_call!(retry_client, call, describe_worker) + } "DeprecateNamespace" => rpc_call!(retry_client, call, deprecate_namespace), "DescribeNamespace" => rpc_call!(retry_client, call, describe_namespace), "DescribeSchedule" => rpc_call!(retry_client, call, describe_schedule), @@ -448,6 +451,9 @@ async fn client_invoke_workflow_service( "SetWorkerDeploymentCurrentVersion" => { rpc_call!(retry_client, call, set_worker_deployment_current_version) } + "SetWorkerDeploymentManager" => { + rpc_call!(retry_client, call, set_worker_deployment_manager) + } "SetWorkerDeploymentRampingVersion" => { rpc_call!(retry_client, call, set_worker_deployment_ramping_version) } From c615bb401c9bae613606cf10c1b676b4a0aacd79 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Wed, 29 Oct 2025 20:07:03 -0400 Subject: [PATCH 04/29] add runtime logger --- packages/test/src/test-runtime.ts | 4 ++-- packages/test/src/test-worker-lifecycle.ts | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/test/src/test-runtime.ts b/packages/test/src/test-runtime.ts index 405e0a90b..932fc9b04 100644 --- a/packages/test/src/test-runtime.ts +++ b/packages/test/src/test-runtime.ts @@ -55,8 +55,8 @@ if (RUN_INTEGRATION_TESTS) { // Runtime configuration was installed, creating a new Worker after Runtime shutdown we would fallback // to the default configuration (127.0.0.1) which is surprising behavior. test.serial('Runtime.install() remembers installed options after it has been shut down', async (t) => { - const logger = new DefaultLogger('DEBUG'); - Runtime.install({ logger }); + const logger = new DefaultLogger('TRACE'); + Runtime.install({ logger, telemetryOptions: { logging: { filter: { core: 'TRACE' } } } }); { const runtime = Runtime.instance(); t.is(runtime.options.logger, logger); diff --git a/packages/test/src/test-worker-lifecycle.ts b/packages/test/src/test-worker-lifecycle.ts index e6e98f548..ef41fe1fa 100644 --- a/packages/test/src/test-worker-lifecycle.ts +++ b/packages/test/src/test-worker-lifecycle.ts @@ -7,7 +7,7 @@ import { setTimeout } from 'timers/promises'; import { randomUUID } from 'crypto'; import test from 'ava'; -import { Runtime, PromiseCompletionTimeoutError } from '@temporalio/worker'; +import { Runtime, PromiseCompletionTimeoutError, makeTelemetryFilterString, DefaultLogger } from '@temporalio/worker'; import { TransportError, UnexpectedError } from '@temporalio/worker/lib/errors'; import { Client } from '@temporalio/client'; import { RUN_INTEGRATION_TESTS, Worker } from './helpers'; @@ -35,6 +35,12 @@ if (RUN_INTEGRATION_TESTS) { }); test.serial("Worker.runUntil doesn't hang if provided promise survives to Worker's shutdown", async (t) => { + const logger = new DefaultLogger('TRACE'); + t.is(Runtime._instance, undefined); + Runtime.install({ + telemetryOptions: { tracingFilter: makeTelemetryFilterString({ core: 'TRACE' }) }, + logger, + }); const worker = await Worker.create({ ...defaultOptions, taskQueue: t.title.replace(/ /g, '_'), From 97769a8d363192ed9bf4eb353a3cb43310c0c789 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Wed, 29 Oct 2025 20:27:27 -0400 Subject: [PATCH 05/29] enable logging for bridge test --- packages/test/src/test-bridge.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/test/src/test-bridge.ts b/packages/test/src/test-bridge.ts index df14e0183..3f31379b1 100644 --- a/packages/test/src/test-bridge.ts +++ b/packages/test/src/test-bridge.ts @@ -234,7 +234,7 @@ const GenericConfigs = { basic: { logExporter: { type: 'console', - filter: 'ERROR', + filter: 'TRACE', }, telemetry: { metricPrefix: 'test', From 18e9c8d1bed54e5c40ad1f5160677a1ed847e96f Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Wed, 29 Oct 2025 21:16:48 -0400 Subject: [PATCH 06/29] add runtime drop logging --- packages/core-bridge/src/runtime.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/core-bridge/src/runtime.rs b/packages/core-bridge/src/runtime.rs index 1c6a8e329..45a2fc984 100644 --- a/packages/core-bridge/src/runtime.rs +++ b/packages/core-bridge/src/runtime.rs @@ -3,7 +3,7 @@ use std::{sync::Arc, time::Duration}; use anyhow::Context as _; use futures::channel::mpsc::Receiver; use neon::prelude::*; -use tracing::{Instrument, warn}; +use tracing::{Instrument, trace, warn}; use temporal_sdk_core::{ CoreRuntime, RuntimeOptionsBuilder, TokioRuntimeBuilder, @@ -136,6 +136,7 @@ pub fn runtime_new( /// runtimes at a high pace, e.g. during tests execution. #[js_function] pub fn runtime_shutdown(runtime: OpaqueInboundHandle) -> BridgeResult<()> { + trace!("dropping runtime"); std::mem::drop(runtime.take()?); Ok(()) } From 2f81432ef28ac5b7a02fa6b365790f86bbf99d7b Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Thu, 30 Oct 2025 07:48:15 -0400 Subject: [PATCH 07/29] bump log filter --- packages/core-bridge/src/runtime.rs | 4 ++-- packages/test/src/test-bridge.ts | 2 +- packages/test/src/test-runtime.ts | 4 ++-- packages/test/src/test-worker-lifecycle.ts | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/core-bridge/src/runtime.rs b/packages/core-bridge/src/runtime.rs index 45a2fc984..cb99815ff 100644 --- a/packages/core-bridge/src/runtime.rs +++ b/packages/core-bridge/src/runtime.rs @@ -3,7 +3,7 @@ use std::{sync::Arc, time::Duration}; use anyhow::Context as _; use futures::channel::mpsc::Receiver; use neon::prelude::*; -use tracing::{Instrument, trace, warn}; +use tracing::{Instrument, debug, warn}; use temporal_sdk_core::{ CoreRuntime, RuntimeOptionsBuilder, TokioRuntimeBuilder, @@ -136,7 +136,7 @@ pub fn runtime_new( /// runtimes at a high pace, e.g. during tests execution. #[js_function] pub fn runtime_shutdown(runtime: OpaqueInboundHandle) -> BridgeResult<()> { - trace!("dropping runtime"); + debug!("dropping runtime"); std::mem::drop(runtime.take()?); Ok(()) } diff --git a/packages/test/src/test-bridge.ts b/packages/test/src/test-bridge.ts index 3f31379b1..82230963f 100644 --- a/packages/test/src/test-bridge.ts +++ b/packages/test/src/test-bridge.ts @@ -234,7 +234,7 @@ const GenericConfigs = { basic: { logExporter: { type: 'console', - filter: 'TRACE', + filter: 'DEBUG', }, telemetry: { metricPrefix: 'test', diff --git a/packages/test/src/test-runtime.ts b/packages/test/src/test-runtime.ts index 932fc9b04..41a3cd947 100644 --- a/packages/test/src/test-runtime.ts +++ b/packages/test/src/test-runtime.ts @@ -55,8 +55,8 @@ if (RUN_INTEGRATION_TESTS) { // Runtime configuration was installed, creating a new Worker after Runtime shutdown we would fallback // to the default configuration (127.0.0.1) which is surprising behavior. test.serial('Runtime.install() remembers installed options after it has been shut down', async (t) => { - const logger = new DefaultLogger('TRACE'); - Runtime.install({ logger, telemetryOptions: { logging: { filter: { core: 'TRACE' } } } }); + const logger = new DefaultLogger('DEBUG'); + Runtime.install({ logger, telemetryOptions: { logging: { filter: { core: 'DEBUG' } } } }); { const runtime = Runtime.instance(); t.is(runtime.options.logger, logger); diff --git a/packages/test/src/test-worker-lifecycle.ts b/packages/test/src/test-worker-lifecycle.ts index ef41fe1fa..b85486113 100644 --- a/packages/test/src/test-worker-lifecycle.ts +++ b/packages/test/src/test-worker-lifecycle.ts @@ -35,10 +35,10 @@ if (RUN_INTEGRATION_TESTS) { }); test.serial("Worker.runUntil doesn't hang if provided promise survives to Worker's shutdown", async (t) => { - const logger = new DefaultLogger('TRACE'); + const logger = new DefaultLogger('DEBUG'); t.is(Runtime._instance, undefined); Runtime.install({ - telemetryOptions: { tracingFilter: makeTelemetryFilterString({ core: 'TRACE' }) }, + telemetryOptions: { tracingFilter: makeTelemetryFilterString({ core: 'DEBUG' }) }, logger, }); const worker = await Worker.create({ From 7b5233c4581b61c987f8c6b65dccf3924c22816d Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Thu, 30 Oct 2025 09:04:24 -0400 Subject: [PATCH 08/29] only run hanging test --- packages/test/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/test/package.json b/packages/test/package.json index 023d01308..3a56fd702 100644 --- a/packages/test/package.json +++ b/packages/test/package.json @@ -7,7 +7,7 @@ "build": "npm-run-all build:protos build:ts", "build:ts": "tsc --build", "build:protos": "node ./scripts/compile-proto.js", - "test": "ava ./lib/test-*.js", + "test": "ava ./lib/test-bridge.js -m \"Stopping Worker after creating another runtime doesn't fail\"", "test.watch": "ava --watch ./lib/test-*.js" }, "ava": { From e3edef8cc8c3916b77ed5ffa8a14b2420704c07f Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Thu, 30 Oct 2025 10:20:38 -0400 Subject: [PATCH 09/29] better logging --- packages/core-bridge/src/client.rs | 57 ++++---- packages/core-bridge/src/runtime.rs | 36 +++++ packages/core-bridge/src/worker.rs | 203 ++++++++++++++++------------ 3 files changed, 181 insertions(+), 115 deletions(-) diff --git a/packages/core-bridge/src/client.rs b/packages/core-bridge/src/client.rs index b9bbd405f..aa47102fd 100644 --- a/packages/core-bridge/src/client.rs +++ b/packages/core-bridge/src/client.rs @@ -55,33 +55,36 @@ pub fn client_new( let runtime = runtime.borrow()?.core_runtime.clone(); let config: CoreClientOptions = config.try_into()?; - runtime.clone().future_to_promise(async move { - let metric_meter = runtime.clone().telemetry().get_temporal_metric_meter(); - let res = config.connect_no_namespace(metric_meter).await; - - let core_client = match res { - Ok(core_client) => core_client, - Err(ClientInitError::InvalidHeaders(e)) => Err(BridgeError::TypeError { - message: format!("Invalid metadata key: {e}"), - field: None, - })?, - Err(ClientInitError::SystemInfoCallError(e)) => Err(BridgeError::TransportError( - format!("Failed to call GetSystemInfo: {e}"), - ))?, - Err(ClientInitError::TonicTransportError(e)) => { - Err(BridgeError::TransportError(format!("{e:?}")))? - } - Err(ClientInitError::InvalidUri(e)) => Err(BridgeError::TypeError { - message: e.to_string(), - field: None, - })?, - }; - - Ok(OpaqueOutboundHandle::new(Client { - core_runtime: runtime, - core_client, - })) - }) + runtime.clone().future_to_promise_named( + async move { + let metric_meter = runtime.clone().telemetry().get_temporal_metric_meter(); + let res = config.connect_no_namespace(metric_meter).await; + + let core_client = match res { + Ok(core_client) => core_client, + Err(ClientInitError::InvalidHeaders(e)) => Err(BridgeError::TypeError { + message: format!("Invalid metadata key: {e}"), + field: None, + })?, + Err(ClientInitError::SystemInfoCallError(e)) => Err(BridgeError::TransportError( + format!("Failed to call GetSystemInfo: {e}"), + ))?, + Err(ClientInitError::TonicTransportError(e)) => { + Err(BridgeError::TransportError(format!("{e:?}")))? + } + Err(ClientInitError::InvalidUri(e)) => Err(BridgeError::TypeError { + message: e.to_string(), + field: None, + })?, + }; + + Ok(OpaqueOutboundHandle::new(Client { + core_runtime: runtime, + core_client, + })) + }, + "client_new", + ) } /// Update a Client's HTTP request headers diff --git a/packages/core-bridge/src/runtime.rs b/packages/core-bridge/src/runtime.rs index cb99815ff..de5354c15 100644 --- a/packages/core-bridge/src/runtime.rs +++ b/packages/core-bridge/src/runtime.rs @@ -167,6 +167,15 @@ pub trait RuntimeExt { F: Future> + Send + 'static, R: TryIntoJs + Send + 'static; + fn future_to_promise_named( + &self, + future: F, + caller: &'static str, + ) -> BridgeResult> + where + F: Future> + Send + 'static, + R: TryIntoJs + Send + 'static; + /// Spawn a future on the Tokio runtime, and let it run to completion without waiting for it to /// complete. Should any error occur, we'll try to send them to this Runtime's logger, but may /// end up or silently dropping entries in some extreme cases. @@ -187,6 +196,21 @@ impl RuntimeExt for CoreRuntime { ))) } + fn future_to_promise_named( + &self, + future: F, + caller: &'static str, + ) -> BridgeResult> + where + F: Future> + Send + 'static, + R: TryIntoJs + Send + 'static, + { + enter_sync!(self); + Ok(BridgeFuture::new(Box::pin(future.instrument( + tracing::info_span!("future_to_promise_named", caller), + )))) + } + fn spawn_and_forget(&self, future: F) where F: Future> + Send + 'static, @@ -216,6 +240,18 @@ impl RuntimeExt for Arc { { self.as_ref().spawn_and_forget(future); } + + fn future_to_promise_named( + &self, + future: F, + caller: &'static str, + ) -> BridgeResult> + where + F: Future> + Send + 'static, + R: TryIntoJs + Send + 'static, + { + self.as_ref().future_to_promise_named(future, caller) + } } //////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/packages/core-bridge/src/worker.rs b/packages/core-bridge/src/worker.rs index a456b92a7..d7749abda 100644 --- a/packages/core-bridge/src/worker.rs +++ b/packages/core-bridge/src/worker.rs @@ -103,12 +103,15 @@ pub fn worker_validate(worker: OpaqueInboundHandle) -> BridgeResult Ok(task.encode_to_vec()), - Err(err) => match err { - PollError::ShutDown => Err(BridgeError::WorkerShutdown)?, - PollError::TonicError(status) => { - Err(BridgeError::TransportError(status.message().to_string()))? - } - }, - } - }) + match result { + Ok(task) => Ok(task.encode_to_vec()), + Err(err) => match err { + PollError::ShutDown => Err(BridgeError::WorkerShutdown)?, + PollError::TonicError(status) => { + Err(BridgeError::TransportError(status.message().to_string()))? + } + }, + } + }, + "worker_poll_workflow_activation", + ) } /// Submit a workflow activation completion to core. @@ -154,21 +160,24 @@ pub fn worker_complete_workflow_activation( let worker = worker_ref.core_worker.clone(); let runtime = worker_ref.core_runtime.clone(); - runtime.future_to_promise(async move { - worker - .complete_workflow_activation(workflow_completion) - .await - .map_err(|err| match err { - CompleteWfError::MalformedWorkflowCompletion { reason, run_id } => { - BridgeError::TypeError { - field: None, - message: format!( - "Malformed Workflow Completion: {reason:?} for RunID={run_id}" - ), + runtime.future_to_promise_named( + async move { + worker + .complete_workflow_activation(workflow_completion) + .await + .map_err(|err| match err { + CompleteWfError::MalformedWorkflowCompletion { reason, run_id } => { + BridgeError::TypeError { + field: None, + message: format!( + "Malformed Workflow Completion: {reason:?} for RunID={run_id}" + ), + } } - } - }) - }) + }) + }, + "worker_complete_workflow_activation", + ) } /// Initiate a single activity task poll request. @@ -181,19 +190,22 @@ pub fn worker_poll_activity_task( let worker = worker_ref.core_worker.clone(); let runtime = worker_ref.core_runtime.clone(); - runtime.future_to_promise(async move { - let result = worker.poll_activity_task().await; + runtime.future_to_promise_named( + async move { + let result = worker.poll_activity_task().await; - match result { - Ok(task) => Ok(task.encode_to_vec()), - Err(err) => match err { - PollError::ShutDown => Err(BridgeError::WorkerShutdown)?, - PollError::TonicError(status) => { - Err(BridgeError::TransportError(status.message().to_string()))? - } - }, - } - }) + match result { + Ok(task) => Ok(task.encode_to_vec()), + Err(err) => match err { + PollError::ShutDown => Err(BridgeError::WorkerShutdown)?, + PollError::TonicError(status) => { + Err(BridgeError::TransportError(status.message().to_string()))? + } + }, + } + }, + "worker_poll_activity_task", + ) } /// Submit an activity task completion to core. @@ -214,20 +226,23 @@ pub fn worker_complete_activity_task( let worker = worker_ref.core_worker.clone(); let runtime = worker_ref.core_runtime.clone(); - runtime.future_to_promise(async move { - worker - .complete_activity_task(activity_completion) - .await - .map_err(|err| match err { - CompleteActivityError::MalformedActivityCompletion { - reason, - completion: _, - } => BridgeError::TypeError { - field: None, - message: format!("Malformed Activity Completion: {reason:?}"), - }, - }) - }) + runtime.future_to_promise_named( + async move { + worker + .complete_activity_task(activity_completion) + .await + .map_err(|err| match err { + CompleteActivityError::MalformedActivityCompletion { + reason, + completion: _, + } => BridgeError::TypeError { + field: None, + message: format!("Malformed Activity Completion: {reason:?}"), + }, + }) + }, + "worker_complete_activity_task", + ) } /// Submit an activity heartbeat to core. @@ -260,19 +275,22 @@ pub fn worker_poll_nexus_task( let worker = worker_ref.core_worker.clone(); let runtime = worker_ref.core_runtime.clone(); - runtime.future_to_promise(async move { - let result = worker.poll_nexus_task().await; + runtime.future_to_promise_named( + async move { + let result = worker.poll_nexus_task().await; - match result { - Ok(task) => Ok(task.encode_to_vec()), - Err(err) => match err { - PollError::ShutDown => Err(BridgeError::WorkerShutdown)?, - PollError::TonicError(status) => { - Err(BridgeError::TransportError(status.message().to_string()))? - } - }, - } - }) + match result { + Ok(task) => Ok(task.encode_to_vec()), + Err(err) => match err { + PollError::ShutDown => Err(BridgeError::WorkerShutdown)?, + PollError::TonicError(status) => { + Err(BridgeError::TransportError(status.message().to_string()))? + } + }, + } + }, + "worker_poll_nexus_task", + ) } /// Submit an nexus task completion to core. @@ -291,20 +309,25 @@ pub fn worker_complete_nexus_task( let worker = worker_ref.core_worker.clone(); let runtime = worker_ref.core_runtime.clone(); - runtime.future_to_promise(async move { - worker - .complete_nexus_task(nexus_completion) - .await - .map_err(|err| match err { - CompleteNexusError::NexusNotEnabled {} => { - BridgeError::UnexpectedError(format!("{err}")) - } - CompleteNexusError::MalformedNexusCompletion { reason } => BridgeError::TypeError { - field: None, - message: format!("Malformed nexus Completion: {reason:?}"), - }, - }) - }) + runtime.future_to_promise_named( + async move { + worker + .complete_nexus_task(nexus_completion) + .await + .map_err(|err| match err { + CompleteNexusError::NexusNotEnabled {} => { + BridgeError::UnexpectedError(format!("{err}")) + } + CompleteNexusError::MalformedNexusCompletion { reason } => { + BridgeError::TypeError { + field: None, + message: format!("Malformed nexus Completion: {reason:?}"), + } + } + }) + }, + "worker_complete_nexus_task", + ) } /// Request shutdown of the worker. @@ -313,6 +336,7 @@ pub fn worker_complete_nexus_task( /// the loop to ensure graceful shutdown. #[js_function] pub fn worker_initiate_shutdown(worker: OpaqueInboundHandle) -> BridgeResult<()> { + tracing::info!("Typescript initiate worker shutdown"); let worker_ref = worker.borrow()?; worker_ref.core_worker.initiate_shutdown(); Ok(()) @@ -337,10 +361,13 @@ pub fn worker_finalize_shutdown( } })?; - worker_ref.core_runtime.future_to_promise(async move { - worker.finalize_shutdown().await; - Ok(()) - }) + worker_ref.core_runtime.future_to_promise_named( + async move { + worker.finalize_shutdown().await; + Ok(()) + }, + "worker_finalize_shutdown", + ) } impl MutableFinalize for Worker { From 6d83993579805fbd678bfc063f88d929f5f995f5 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Thu, 30 Oct 2025 10:29:30 -0400 Subject: [PATCH 10/29] use additional events from core --- packages/core-bridge/sdk-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core-bridge/sdk-core b/packages/core-bridge/sdk-core index 9e9a46191..93927e2da 160000 --- a/packages/core-bridge/sdk-core +++ b/packages/core-bridge/sdk-core @@ -1 +1 @@ -Subproject commit 9e9a46191656fc9ccd95589dac3552410561d620 +Subproject commit 93927e2daeb97514f31a52bc6defa68dcb7149c0 From d4e8d74586f56724ac7412d9d0815da8bac9bf8d Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Thu, 30 Oct 2025 11:09:45 -0400 Subject: [PATCH 11/29] bump tracing branch --- packages/core-bridge/sdk-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core-bridge/sdk-core b/packages/core-bridge/sdk-core index 93927e2da..1f90164c0 160000 --- a/packages/core-bridge/sdk-core +++ b/packages/core-bridge/sdk-core @@ -1 +1 @@ -Subproject commit 93927e2daeb97514f31a52bc6defa68dcb7149c0 +Subproject commit 1f90164c062177ebcb7b66a2e9ccb41590fbbbe0 From 741bc723c143408b94a57f9cf006e802c6c7ee48 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Thu, 30 Oct 2025 11:19:01 -0400 Subject: [PATCH 12/29] bump to info level --- packages/test/src/test-bridge.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/test/src/test-bridge.ts b/packages/test/src/test-bridge.ts index 82230963f..f5d3aefa7 100644 --- a/packages/test/src/test-bridge.ts +++ b/packages/test/src/test-bridge.ts @@ -234,7 +234,7 @@ const GenericConfigs = { basic: { logExporter: { type: 'console', - filter: 'DEBUG', + filter: 'INFO', }, telemetry: { metricPrefix: 'test', From e07739355d3e04f653d7a6ff09838309d13f7f1f Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Thu, 30 Oct 2025 11:58:30 -0400 Subject: [PATCH 13/29] bump tracing branch --- packages/core-bridge/sdk-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core-bridge/sdk-core b/packages/core-bridge/sdk-core index 1f90164c0..818a1b69c 160000 --- a/packages/core-bridge/sdk-core +++ b/packages/core-bridge/sdk-core @@ -1 +1 @@ -Subproject commit 1f90164c062177ebcb7b66a2e9ccb41590fbbbe0 +Subproject commit 818a1b69c7e8bf1e3b566a39359ae77b1bec3d8e From 203348dfd2458afd6a6def2c492b638cdeda1658 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Thu, 30 Oct 2025 13:18:25 -0400 Subject: [PATCH 14/29] enable more tracing --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab417a0b0..8f33b52d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -210,6 +210,7 @@ jobs: - name: Run Tests run: npm run test env: + TEMPORAL_TRACE_NATIVE_CALLS: true RUN_INTEGRATION_TESTS: true REUSE_V8_CONTEXT: ${{ matrix.reuse-v8-context }} From 6474f2fd68ca83ae049e745ef5dcf3ef75a2918a Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Thu, 30 Oct 2025 13:42:17 -0400 Subject: [PATCH 15/29] bump core --- packages/core-bridge/sdk-core | 2 +- packages/test/src/test-bridge.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/core-bridge/sdk-core b/packages/core-bridge/sdk-core index 818a1b69c..cb27bff9d 160000 --- a/packages/core-bridge/sdk-core +++ b/packages/core-bridge/sdk-core @@ -1 +1 @@ -Subproject commit 818a1b69c7e8bf1e3b566a39359ae77b1bec3d8e +Subproject commit cb27bff9d0152e46e5bd9e31148e0fd96c0ed48c diff --git a/packages/test/src/test-bridge.ts b/packages/test/src/test-bridge.ts index f5d3aefa7..a04a68d2c 100644 --- a/packages/test/src/test-bridge.ts +++ b/packages/test/src/test-bridge.ts @@ -217,6 +217,7 @@ test("Stopping Worker after creating another runtime doesn't fail", async (t) => await native.workerValidate(worker2); const wftPromise2 = native.workerPollWorkflowActivation(worker2); const atPromise2 = native.workerPollActivityTask(worker2); + await setTimeout(100); native.workerInitiateShutdown(worker2); await expectShutdownError(wftPromise2); await expectShutdownError(atPromise2); From eb4e51a85b399d1d0363dc16d808fe70e2644dd5 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Thu, 30 Oct 2025 13:53:15 -0400 Subject: [PATCH 16/29] bump timeout --- packages/test/src/test-bridge.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/test/src/test-bridge.ts b/packages/test/src/test-bridge.ts index a04a68d2c..99c7494b3 100644 --- a/packages/test/src/test-bridge.ts +++ b/packages/test/src/test-bridge.ts @@ -163,6 +163,7 @@ test("Dropping EphemeralServer without shutting it down doesn't hang process", a }); test("Stopping Worker after creating another runtime doesn't fail", async (t) => { + t.timeout(120 * 1000); async function expectShutdownError(taskPromise: Promise) { await t.throwsAsync(taskPromise, { instanceOf: errors.ShutdownError, @@ -217,7 +218,7 @@ test("Stopping Worker after creating another runtime doesn't fail", async (t) => await native.workerValidate(worker2); const wftPromise2 = native.workerPollWorkflowActivation(worker2); const atPromise2 = native.workerPollActivityTask(worker2); - await setTimeout(100); + await setTimeout(3000); native.workerInitiateShutdown(worker2); await expectShutdownError(wftPromise2); await expectShutdownError(atPromise2); From 4ffd5760a8cc25b5aa2b23b408e27cf307e6f6dd Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Thu, 30 Oct 2025 14:06:47 -0400 Subject: [PATCH 17/29] remove timeout --- packages/test/src/test-bridge.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/test/src/test-bridge.ts b/packages/test/src/test-bridge.ts index 99c7494b3..e2b8c96fa 100644 --- a/packages/test/src/test-bridge.ts +++ b/packages/test/src/test-bridge.ts @@ -218,7 +218,7 @@ test("Stopping Worker after creating another runtime doesn't fail", async (t) => await native.workerValidate(worker2); const wftPromise2 = native.workerPollWorkflowActivation(worker2); const atPromise2 = native.workerPollActivityTask(worker2); - await setTimeout(3000); + // await setTimeout(3000); native.workerInitiateShutdown(worker2); await expectShutdownError(wftPromise2); await expectShutdownError(atPromise2); From 53d3660ca139619716c23e8e55688b6c0a1f77d1 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Thu, 30 Oct 2025 14:21:16 -0400 Subject: [PATCH 18/29] add back heartbeat --- packages/core-bridge/sdk-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core-bridge/sdk-core b/packages/core-bridge/sdk-core index cb27bff9d..ebbe50caa 160000 --- a/packages/core-bridge/sdk-core +++ b/packages/core-bridge/sdk-core @@ -1 +1 @@ -Subproject commit cb27bff9d0152e46e5bd9e31148e0fd96c0ed48c +Subproject commit ebbe50caa5c9858cfd1fa40f13620b3d0dc70e29 From a049ed0e62ba1cab0c45d8bc596ff9e08edc57f2 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Thu, 30 Oct 2025 15:02:44 -0400 Subject: [PATCH 19/29] disable heartbeat --- packages/core-bridge/sdk-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core-bridge/sdk-core b/packages/core-bridge/sdk-core index ebbe50caa..e0bfdd82e 160000 --- a/packages/core-bridge/sdk-core +++ b/packages/core-bridge/sdk-core @@ -1 +1 @@ -Subproject commit ebbe50caa5c9858cfd1fa40f13620b3d0dc70e29 +Subproject commit e0bfdd82e1718159e521a90cd11ad61040fd2b37 From a7282cc828ed46380502ca47dd5cf8e12e4760bb Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Thu, 30 Oct 2025 15:13:28 -0400 Subject: [PATCH 20/29] bump to instrument heartbeating --- packages/core-bridge/sdk-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core-bridge/sdk-core b/packages/core-bridge/sdk-core index e0bfdd82e..2559e9304 160000 --- a/packages/core-bridge/sdk-core +++ b/packages/core-bridge/sdk-core @@ -1 +1 @@ -Subproject commit e0bfdd82e1718159e521a90cd11ad61040fd2b37 +Subproject commit 2559e93046bf9cdbc375801fe715b49c847b41ca From 20ead68f4588ccb4078f0f7f7193139794af21e2 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Thu, 30 Oct 2025 15:21:04 -0400 Subject: [PATCH 21/29] bump core --- packages/core-bridge/sdk-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core-bridge/sdk-core b/packages/core-bridge/sdk-core index 2559e9304..07c7bf370 160000 --- a/packages/core-bridge/sdk-core +++ b/packages/core-bridge/sdk-core @@ -1 +1 @@ -Subproject commit 2559e93046bf9cdbc375801fe715b49c847b41ca +Subproject commit 07c7bf370baa7cd19949eb7d6cc5fb9c68664f1b From ca73ffbc11767140a48ee564ab0f5e5d200d2600 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Thu, 30 Oct 2025 15:28:12 -0400 Subject: [PATCH 22/29] bump core with warn on heartbeat missing --- packages/core-bridge/sdk-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core-bridge/sdk-core b/packages/core-bridge/sdk-core index 07c7bf370..b6da12723 160000 --- a/packages/core-bridge/sdk-core +++ b/packages/core-bridge/sdk-core @@ -1 +1 @@ -Subproject commit 07c7bf370baa7cd19949eb7d6cc5fb9c68664f1b +Subproject commit b6da1272337bad08eba3dce55912a074570213ee From a02eac629eb6ed3a87a31d1203d64112aa7b09cd Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Thu, 30 Oct 2025 15:41:38 -0400 Subject: [PATCH 23/29] bump core to disable heartbeat --- packages/core-bridge/sdk-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core-bridge/sdk-core b/packages/core-bridge/sdk-core index b6da12723..e765ab724 160000 --- a/packages/core-bridge/sdk-core +++ b/packages/core-bridge/sdk-core @@ -1 +1 @@ -Subproject commit b6da1272337bad08eba3dce55912a074570213ee +Subproject commit e765ab724e2900ebb5f2b80a98c710974c7e5d83 From 99493cb19b3828de257e91bc0205f6970119b269 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Thu, 30 Oct 2025 16:10:16 -0400 Subject: [PATCH 24/29] enable features for ci server --- .github/workflows/ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8f33b52d7..292729699 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -205,6 +205,14 @@ jobs: --db-filename temporal.sqlite \ --sqlite-pragma journal_mode=WAL \ --sqlite-pragma synchronous=OFF \ + --dynamic-config-value system.enableEagerWorkflowStart=true \ + --dynamic-config-value system.enableNexus=true \ + --dynamic-config-value frontend.workerVersioningWorkflowAPIs=true \ + --dynamic-config-value frontend.workerVersioningDataAPIs=true \ + --dynamic-config-value system.enableDeploymentVersions=true \ + --dynamic-config-value component.nexusoperations.recordCancelRequestCompletionEvents=true \ + --dynamic-config-value frontend.WorkerHeartbeatsEnabled=true \ + --dynamic-config-value frontend.ListWorkersEnabled=true \ --headless &> ./devserver.log & - name: Run Tests From 60d086471ff23818326208cb8bbbdb0a1c3b0300 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Thu, 30 Oct 2025 16:10:49 -0400 Subject: [PATCH 25/29] reenable heartbeat from core --- packages/core-bridge/sdk-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core-bridge/sdk-core b/packages/core-bridge/sdk-core index e765ab724..41d1b7548 160000 --- a/packages/core-bridge/sdk-core +++ b/packages/core-bridge/sdk-core @@ -1 +1 @@ -Subproject commit e765ab724e2900ebb5f2b80a98c710974c7e5d83 +Subproject commit 41d1b75489c63c45331d065b600f0d7501414c80 From fb87c2161918416dcdc2e04a41f45dc9589d8f7a Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Thu, 30 Oct 2025 16:41:04 -0400 Subject: [PATCH 26/29] add additional heartbeat tracing --- packages/core-bridge/sdk-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core-bridge/sdk-core b/packages/core-bridge/sdk-core index 41d1b7548..cc7c67a41 160000 --- a/packages/core-bridge/sdk-core +++ b/packages/core-bridge/sdk-core @@ -1 +1 @@ -Subproject commit 41d1b75489c63c45331d065b600f0d7501414c80 +Subproject commit cc7c67a417f0db9ce495d9b5b951f2e5cb82ea37 From fc3bb92e3ff9f93b1909e4a8f6136865dc872499 Mon Sep 17 00:00:00 2001 From: James Watkins-Harvey Date: Thu, 30 Oct 2025 17:20:38 -0400 Subject: [PATCH 27/29] Set heartbeat_internal to None --- packages/core-bridge/src/runtime.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core-bridge/src/runtime.rs b/packages/core-bridge/src/runtime.rs index de5354c15..616c82f65 100644 --- a/packages/core-bridge/src/runtime.rs +++ b/packages/core-bridge/src/runtime.rs @@ -65,6 +65,7 @@ pub fn runtime_new( let mut core_runtime = CoreRuntime::new( RuntimeOptionsBuilder::default() .telemetry_options(telemetry_options) + .heartbeat_interval(None) .build() .unwrap(), TokioRuntimeBuilder::default(), From c44f0b127b2b8eefe433491c7c41b9179fa92fd3 Mon Sep 17 00:00:00 2001 From: James Watkins-Harvey Date: Thu, 30 Oct 2025 17:47:13 -0400 Subject: [PATCH 28/29] Reenable all tests --- .github/workflows/ci.yml | 12 ++++++------ packages/test/package.json | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 292729699..eacb66ef7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,12 +56,12 @@ jobs: run: shell: bash steps: - - name: 'Checkout code' + - name: "Checkout code" uses: actions/checkout@v4 with: submodules: recursive - - name: 'Cache index.node' + - name: "Cache index.node" id: cached-artifact uses: actions/cache@v4 with: @@ -73,7 +73,7 @@ jobs: uses: arduino/setup-protoc@v3 with: # TODO: Upgrade proto once https://github.com/arduino/setup-protoc/issues/99 is fixed - version: '23.x' + version: "23.x" repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Upgrade Rust to latest stable @@ -86,7 +86,7 @@ jobs: workspaces: packages/core-bridge -> target prefix-key: corebridge-buildcache-debug shared-key: ${{ matrix.platform }} - env-vars: '' + env-vars: "" save-if: ${{ env.IS_MAIN_OR_RELEASE == 'true' }} - name: Compile rust code @@ -144,7 +144,7 @@ jobs: - name: Set git config run: git config --global core.autocrlf false - - name: 'Checkout code' + - name: "Checkout code" uses: actions/checkout@v4 with: submodules: recursive @@ -218,7 +218,7 @@ jobs: - name: Run Tests run: npm run test env: - TEMPORAL_TRACE_NATIVE_CALLS: true + TEMPORAL_TRACE_NATIVE_CALLS_XXXXX: true RUN_INTEGRATION_TESTS: true REUSE_V8_CONTEXT: ${{ matrix.reuse-v8-context }} diff --git a/packages/test/package.json b/packages/test/package.json index 3a56fd702..186cb18e3 100644 --- a/packages/test/package.json +++ b/packages/test/package.json @@ -7,7 +7,8 @@ "build": "npm-run-all build:protos build:ts", "build:ts": "tsc --build", "build:protos": "node ./scripts/compile-proto.js", - "test": "ava ./lib/test-bridge.js -m \"Stopping Worker after creating another runtime doesn't fail\"", + "test": "ava ./lib/test-*.js", + "test:xxxx": "ava ./lib/test-bridge.js -m \"Stopping Worker after creating another runtime doesn't fail\"", "test.watch": "ava --watch ./lib/test-*.js" }, "ava": { From f2784085722e30ac1b0133b3ee6ce79d0a0a29e6 Mon Sep 17 00:00:00 2001 From: James Watkins-Harvey Date: Thu, 30 Oct 2025 18:32:43 -0400 Subject: [PATCH 29/29] Enable only runtime tests --- .github/workflows/ci.yml | 203 +++++++++++++++---------------- packages/test/package.json | 3 +- packages/test/src/test-bridge.ts | 2 - 3 files changed, 102 insertions(+), 106 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eacb66ef7..7fe73a670 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -218,7 +218,7 @@ jobs: - name: Run Tests run: npm run test env: - TEMPORAL_TRACE_NATIVE_CALLS_XXXXX: true + TEMPORAL_TRACE_NATIVE_CALLS: true RUN_INTEGRATION_TESTS: true REUSE_V8_CONTEXT: ${{ matrix.reuse-v8-context }} @@ -238,104 +238,103 @@ jobs: TEMPORAL_CLOUD_OPS_TEST_NAMESPACE: ${{ vars.TEMPORAL_CLIENT_NAMESPACE }} TEMPORAL_CLOUD_OPS_TEST_API_KEY: ${{ secrets.TEMPORAL_CLIENT_CLOUD_API_KEY }} TEMPORAL_CLOUD_OPS_TEST_API_VERSION: 2024-05-13-00 - - # FIXME: Move samples tests to a custom activity - # Sample 1: hello-world to local server - - name: Instantiate sample project using verdaccio artifacts - Hello World - run: | - node scripts/init-from-verdaccio.js --registry-dir ${{ steps.tmp-dir.outputs.dir }}/npm-registry --sample https://github.com/temporalio/samples-typescript/tree/main/hello-world --target-dir ${{ steps.tmp-dir.outputs.dir }}/sample-hello-world - node scripts/test-example.js --work-dir "${{ steps.tmp-dir.outputs.dir }}/sample-hello-world" - - # Sample 2: hello-world-mtls to cloud server - - name: Instantiate sample project using verdaccio artifacts - Hello World MTLS - run: | - if [ -z "$TEMPORAL_ADDRESS" ] || [ -z "$TEMPORAL_NAMESPACE" ] || [ -z "$TEMPORAL_CLIENT_CERT" ] || [ -z "$TEMPORAL_CLIENT_KEY" ]; then - echo "Skipping hello-world-mtls sample test as required environment variables are not set" - exit 0 - fi - - node scripts/create-certs-dir.js ${{ steps.tmp-dir.outputs.dir }}/certs - node scripts/init-from-verdaccio.js --registry-dir ${{ steps.tmp-dir.outputs.dir }}/npm-registry --sample https://github.com/temporalio/samples-typescript/tree/main/hello-world-mtls --target-dir ${{ steps.tmp-dir.outputs.dir }}/sample-hello-world-mtls - node scripts/test-example.js --work-dir "${{ steps.tmp-dir.outputs.dir }}/sample-hello-world-mtls" - env: - # These env vars are used by the hello-world-mtls sample - TEMPORAL_ADDRESS: ${{ vars.TEMPORAL_CLIENT_NAMESPACE }}.tmprl.cloud:7233 - TEMPORAL_NAMESPACE: ${{ vars.TEMPORAL_CLIENT_NAMESPACE }} - TEMPORAL_CLIENT_CERT: ${{ secrets.TEMPORAL_CLIENT_CERT }} - TEMPORAL_CLIENT_KEY: ${{ secrets.TEMPORAL_CLIENT_KEY }} - TEMPORAL_TASK_QUEUE: ${{ format('tssdk-ci-{0}-{1}-sample-hello-world-mtls-{2}-{3}', matrix.platform, matrix.node, github.run_id, github.run_attempt) }} - - TEMPORAL_CLIENT_CERT_PATH: ${{ steps.tmp-dir.outputs.dir }}/certs/client.pem - TEMPORAL_CLIENT_KEY_PATH: ${{ steps.tmp-dir.outputs.dir }}/certs/client.key - - - name: Destroy certs dir - if: always() - run: rm -rf ${{ steps.tmp-dir.outputs.dir }}/certs - continue-on-error: true - - # Sample 3: fetch-esm to local server - - name: Instantiate sample project using verdaccio artifacts - Fetch ESM - run: | - node scripts/init-from-verdaccio.js --registry-dir ${{ steps.tmp-dir.outputs.dir }}/npm-registry --sample https://github.com/temporalio/samples-typescript/tree/main/fetch-esm --target-dir ${{ steps.tmp-dir.outputs.dir }}/sample-fetch-esm - node scripts/test-example.js --work-dir "${{ steps.tmp-dir.outputs.dir }}/sample-fetch-esm" - - # End samples - - - name: Upload NPM logs - uses: actions/upload-artifact@v4 - if: failure() || cancelled() - with: - name: integration-tests-${{ matrix.platform }}-node${{ matrix.node }}-${{ matrix.reuse-v8-context && 'reuse' || 'noreuse' }}-logs - path: ${{ startsWith(matrix.platform, 'windows') && 'C:\\npm\\_logs\\' || '~/.npm/_logs/' }} - - - name: Upload Dev Server logs - uses: actions/upload-artifact@v4 - if: failure() || cancelled() - with: - name: integration-tests-${{ matrix.platform }}-node${{ matrix.node }}-${{ matrix.reuse-v8-context && 'reuse' || 'noreuse' }}-devserver-logs - path: ${{ steps.tmp-dir.outputs.dir }}/devserver.log - - conventions: - name: Lint and Prune - uses: ./.github/workflows/conventions.yml - - # Runs the features repo tests with this repo's current SDK code - # FIXME: Update this job to reuse native build artifacts from compile-native-binaries - features-tests: - name: Features Tests - uses: temporalio/features/.github/workflows/typescript.yaml@main - with: - typescript-repo-path: ${{github.event.pull_request.head.repo.full_name}} - version: ${{github.event.pull_request.head.ref}} - version-is-repo-ref: true - features-repo-ref: main - - stress-tests-no-reuse-context: - name: Stress Tests (No Reuse V8 Context) - # FIXME: Update this job to reuse native build artifacts from compile-native-binaries - uses: ./.github/workflows/stress.yml - with: - test-type: ci-stress - test-timeout-minutes: 20 - reuse-v8-context: false - - stress-tests-reuse-context: - name: Stress Tests (Reuse V8 Context) - # FIXME: Update this job to reuse native build artifacts from compile-native-binaries - uses: ./.github/workflows/stress.yml - with: - test-type: ci-stress - test-timeout-minutes: 20 - reuse-v8-context: true - - docs: - name: Build Docs - uses: ./.github/workflows/docs.yml - with: - # Can't publish from forks, as secrets won't be available - publish_target: ${{ vars.IS_TEMPORALIO_SDK_TYPESCRIPT_REPO == 'true' && 'draft' || '' }} - secrets: - ALGOLIA_API_KEY: ${{ secrets.ALGOLIA_API_KEY }} - VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} - VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }} - VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} +# # FIXME: Move samples tests to a custom activity +# # Sample 1: hello-world to local server +# - name: Instantiate sample project using verdaccio artifacts - Hello World +# run: | +# node scripts/init-from-verdaccio.js --registry-dir ${{ steps.tmp-dir.outputs.dir }}/npm-registry --sample https://github.com/temporalio/samples-typescript/tree/main/hello-world --target-dir ${{ steps.tmp-dir.outputs.dir }}/sample-hello-world +# node scripts/test-example.js --work-dir "${{ steps.tmp-dir.outputs.dir }}/sample-hello-world" +# +# # Sample 2: hello-world-mtls to cloud server +# - name: Instantiate sample project using verdaccio artifacts - Hello World MTLS +# run: | +# if [ -z "$TEMPORAL_ADDRESS" ] || [ -z "$TEMPORAL_NAMESPACE" ] || [ -z "$TEMPORAL_CLIENT_CERT" ] || [ -z "$TEMPORAL_CLIENT_KEY" ]; then +# echo "Skipping hello-world-mtls sample test as required environment variables are not set" +# exit 0 +# fi +# +# node scripts/create-certs-dir.js ${{ steps.tmp-dir.outputs.dir }}/certs +# node scripts/init-from-verdaccio.js --registry-dir ${{ steps.tmp-dir.outputs.dir }}/npm-registry --sample https://github.com/temporalio/samples-typescript/tree/main/hello-world-mtls --target-dir ${{ steps.tmp-dir.outputs.dir }}/sample-hello-world-mtls +# node scripts/test-example.js --work-dir "${{ steps.tmp-dir.outputs.dir }}/sample-hello-world-mtls" +# env: +# # These env vars are used by the hello-world-mtls sample +# TEMPORAL_ADDRESS: ${{ vars.TEMPORAL_CLIENT_NAMESPACE }}.tmprl.cloud:7233 +# TEMPORAL_NAMESPACE: ${{ vars.TEMPORAL_CLIENT_NAMESPACE }} +# TEMPORAL_CLIENT_CERT: ${{ secrets.TEMPORAL_CLIENT_CERT }} +# TEMPORAL_CLIENT_KEY: ${{ secrets.TEMPORAL_CLIENT_KEY }} +# TEMPORAL_TASK_QUEUE: ${{ format('tssdk-ci-{0}-{1}-sample-hello-world-mtls-{2}-{3}', matrix.platform, matrix.node, github.run_id, github.run_attempt) }} +# +# TEMPORAL_CLIENT_CERT_PATH: ${{ steps.tmp-dir.outputs.dir }}/certs/client.pem +# TEMPORAL_CLIENT_KEY_PATH: ${{ steps.tmp-dir.outputs.dir }}/certs/client.key +# +# - name: Destroy certs dir +# if: always() +# run: rm -rf ${{ steps.tmp-dir.outputs.dir }}/certs +# continue-on-error: true +# +# # Sample 3: fetch-esm to local server +# - name: Instantiate sample project using verdaccio artifacts - Fetch ESM +# run: | +# node scripts/init-from-verdaccio.js --registry-dir ${{ steps.tmp-dir.outputs.dir }}/npm-registry --sample https://github.com/temporalio/samples-typescript/tree/main/fetch-esm --target-dir ${{ steps.tmp-dir.outputs.dir }}/sample-fetch-esm +# node scripts/test-example.js --work-dir "${{ steps.tmp-dir.outputs.dir }}/sample-fetch-esm" +# +# # End samples +# +# - name: Upload NPM logs +# uses: actions/upload-artifact@v4 +# if: failure() || cancelled() +# with: +# name: integration-tests-${{ matrix.platform }}-node${{ matrix.node }}-${{ matrix.reuse-v8-context && 'reuse' || 'noreuse' }}-logs +# path: ${{ startsWith(matrix.platform, 'windows') && 'C:\\npm\\_logs\\' || '~/.npm/_logs/' }} +# +# - name: Upload Dev Server logs +# uses: actions/upload-artifact@v4 +# if: failure() || cancelled() +# with: +# name: integration-tests-${{ matrix.platform }}-node${{ matrix.node }}-${{ matrix.reuse-v8-context && 'reuse' || 'noreuse' }}-devserver-logs +# path: ${{ steps.tmp-dir.outputs.dir }}/devserver.log + +#conventions: +# name: Lint and Prune +# uses: ./.github/workflows/conventions.yml + +# Runs the features repo tests with this repo's current SDK code +# FIXME: Update this job to reuse native build artifacts from compile-native-binaries +#features-tests: +# name: Features Tests +# uses: temporalio/features/.github/workflows/typescript.yaml@main +# with: +# typescript-repo-path: ${{github.event.pull_request.head.repo.full_name}} +# version: ${{github.event.pull_request.head.ref}} +# version-is-repo-ref: true +# features-repo-ref: main + +#stress-tests-no-reuse-context: +# name: Stress Tests (No Reuse V8 Context) +# # FIXME: Update this job to reuse native build artifacts from compile-native-binaries +# uses: ./.github/workflows/stress.yml +# with: +# test-type: ci-stress +# test-timeout-minutes: 20 +# reuse-v8-context: false + +#stress-tests-reuse-context: +# name: Stress Tests (Reuse V8 Context) +# # FIXME: Update this job to reuse native build artifacts from compile-native-binaries +# uses: ./.github/workflows/stress.yml +# with: +# test-type: ci-stress +# test-timeout-minutes: 20 +# reuse-v8-context: true + +#docs: +# name: Build Docs +# uses: ./.github/workflows/docs.yml +# with: +# # Can't publish from forks, as secrets won't be available +# publish_target: ${{ vars.IS_TEMPORALIO_SDK_TYPESCRIPT_REPO == 'true' && 'draft' || '' }} +# secrets: +# ALGOLIA_API_KEY: ${{ secrets.ALGOLIA_API_KEY }} +# VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} +# VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }} +# VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} diff --git a/packages/test/package.json b/packages/test/package.json index 186cb18e3..200b9e300 100644 --- a/packages/test/package.json +++ b/packages/test/package.json @@ -7,8 +7,7 @@ "build": "npm-run-all build:protos build:ts", "build:ts": "tsc --build", "build:protos": "node ./scripts/compile-proto.js", - "test": "ava ./lib/test-*.js", - "test:xxxx": "ava ./lib/test-bridge.js -m \"Stopping Worker after creating another runtime doesn't fail\"", + "test": "ava ./lib/test-runtime.js", "test.watch": "ava --watch ./lib/test-*.js" }, "ava": { diff --git a/packages/test/src/test-bridge.ts b/packages/test/src/test-bridge.ts index e2b8c96fa..f5d3aefa7 100644 --- a/packages/test/src/test-bridge.ts +++ b/packages/test/src/test-bridge.ts @@ -163,7 +163,6 @@ test("Dropping EphemeralServer without shutting it down doesn't hang process", a }); test("Stopping Worker after creating another runtime doesn't fail", async (t) => { - t.timeout(120 * 1000); async function expectShutdownError(taskPromise: Promise) { await t.throwsAsync(taskPromise, { instanceOf: errors.ShutdownError, @@ -218,7 +217,6 @@ test("Stopping Worker after creating another runtime doesn't fail", async (t) => await native.workerValidate(worker2); const wftPromise2 = native.workerPollWorkflowActivation(worker2); const atPromise2 = native.workerPollActivityTask(worker2); - // await setTimeout(3000); native.workerInitiateShutdown(worker2); await expectShutdownError(wftPromise2); await expectShutdownError(atPromise2);