From 6ec786dce8e983bf5cfa244919451a4a201d58d0 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 26 Nov 2025 14:05:57 -0600 Subject: [PATCH 01/57] Started building out RemoteFileSource plugin (#DH-20578) --- plugin/remotefilesource/build.gradle | 14 ++++ plugin/remotefilesource/gradle.properties | 1 + .../RemoteFileSourceCommandResolver.java | 79 +++++++++++++++++++ .../RemoteFileSourceServicePlugin.java | 61 ++++++++++++++ proto/proto-backplane-grpc/Dockerfile | 16 ++++ .../proto/remotefilesource.proto | 36 +++++++++ settings.gradle | 3 + 7 files changed, 210 insertions(+) create mode 100644 plugin/remotefilesource/build.gradle create mode 100644 plugin/remotefilesource/gradle.properties create mode 100644 plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java create mode 100644 plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java create mode 100644 proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto diff --git a/plugin/remotefilesource/build.gradle b/plugin/remotefilesource/build.gradle new file mode 100644 index 00000000000..61e914c8f88 --- /dev/null +++ b/plugin/remotefilesource/build.gradle @@ -0,0 +1,14 @@ +plugins { + id 'io.deephaven.project.register' +} + +dependencies { + implementation project(':plugin') + implementation project(':server') + implementation project(':proto:proto-backplane-grpc') + implementation project(':proto:proto-backplane-grpc-flight') + + compileOnly libs.autoservice + compileOnly libs.jetbrains.annotations + annotationProcessor libs.autoservice.compiler +} \ No newline at end of file diff --git a/plugin/remotefilesource/gradle.properties b/plugin/remotefilesource/gradle.properties new file mode 100644 index 00000000000..c186bbfdde1 --- /dev/null +++ b/plugin/remotefilesource/gradle.properties @@ -0,0 +1 @@ +io.deephaven.project.ProjectType=JAVA_PUBLIC diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java new file mode 100644 index 00000000000..669a6130798 --- /dev/null +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java @@ -0,0 +1,79 @@ +package io.deephaven.remotefilesource; + +import io.deephaven.server.session.CommandResolver; +import io.deephaven.server.session.SessionState; +import io.deephaven.server.session.TicketRouter; +import io.deephaven.server.session.WantsTicketRouter; + +import org.apache.arrow.flight.impl.Flight; +import org.jetbrains.annotations.Nullable; + +import java.nio.ByteBuffer; +import java.util.function.Consumer; + +public class RemoteFileSourceCommandResolver implements CommandResolver, WantsTicketRouter { + @Override + public SessionState.ExportObject flightInfoFor(@Nullable final SessionState session, + final Flight.FlightDescriptor descriptor, + final String logId) { + return null; + } + + @Override + public void forAllFlightInfo(@Nullable final SessionState session, final Consumer visitor) { + // nothing to do + } + + @Override + public String getLogNameFor(final ByteBuffer ticket, final String logId) { + // no tickets + throw new UnsupportedOperationException(); + } + + @Override + public boolean handlesCommand(final Flight.FlightDescriptor descriptor) { + return false; + } + + @Override + public SessionState.ExportBuilder publish(final SessionState session, + final ByteBuffer ticket, + final String logId, + @Nullable final Runnable onPublish) { + // no publishing + throw new UnsupportedOperationException(); + } + + @Override + public SessionState.ExportBuilder publish(final SessionState session, + final Flight.FlightDescriptor descriptor, final String logId, + @Nullable final Runnable onPublish) { + // no publishing + throw new UnsupportedOperationException(); + } + + @Override + public SessionState.ExportObject resolve(@Nullable final SessionState session, + final Flight.FlightDescriptor descriptor, + final String logId) { + return null; + } + + @Override + public SessionState.ExportObject resolve(@Nullable final SessionState session, + final ByteBuffer ticket, + final String logId) { + // no tickets + throw new UnsupportedOperationException(); + } + + @Override + public void setTicketRouter(TicketRouter ticketRouter) { + // not needed + } + + @Override + public byte ticketRoute() { + return 0; + } +} diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java new file mode 100644 index 00000000000..9fcf6bc8505 --- /dev/null +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java @@ -0,0 +1,61 @@ +package io.deephaven.remotefilesource; + +import com.google.auto.service.AutoService; +import com.google.protobuf.InvalidProtocolBufferException; +import io.deephaven.plugin.type.ObjectType; +import io.deephaven.plugin.type.ObjectTypeBase; +import io.deephaven.plugin.type.ObjectCommunicationException; +import io.deephaven.proto.backplane.grpc.RemoteFileSourcePluginFetchRequest; + +import java.nio.ByteBuffer; + +@AutoService(ObjectType.class) +public class RemoteFileSourceServicePlugin extends ObjectTypeBase { + public RemoteFileSourceServicePlugin() {} + + @Override + public String name() { + return "RemoteFileSourceService"; + } + + @Override + public boolean isType(Object object) { + return object instanceof RemoteFileSourceServicePlugin; + } + + @Override + public MessageStream compatibleClientConnection(Object object, MessageStream connection) throws ObjectCommunicationException { + connection.onData(ByteBuffer.allocate(0)); + return new RemoteFileSourceMessageStream(connection); + } + + /** + * A message stream for the RemoteFileSourceService. + */ + private class RemoteFileSourceMessageStream implements MessageStream { + private final MessageStream connection; + + public RemoteFileSourceMessageStream(final MessageStream connection) { + this.connection = connection; + } + + @Override + public void onData(ByteBuffer payload, Object... references) throws ObjectCommunicationException { + final RemoteFileSourcePluginFetchRequest request; + + try { + request = RemoteFileSourcePluginFetchRequest.parseFrom(payload); + } catch (InvalidProtocolBufferException e) { + // There is no identifier here, so we cannot properly return an error that is bound to the request. + // Instead, we throw an Exception causing the server to close the entire MessageStream and + // propagate a general error to the client. + throw new RuntimeException(e); + } + } + + @Override + public void onClose() { + + } + } +} diff --git a/proto/proto-backplane-grpc/Dockerfile b/proto/proto-backplane-grpc/Dockerfile index a95d0465655..0b142c717a0 100644 --- a/proto/proto-backplane-grpc/Dockerfile +++ b/proto/proto-backplane-grpc/Dockerfile @@ -29,6 +29,7 @@ RUN set -eux; \ /includes/deephaven_core/proto/application.proto \ /includes/deephaven_core/proto/inputtable.proto \ /includes/deephaven_core/proto/partitionedtable.proto \ + /includes/deephaven_core/proto/remotefilesource.proto \ /includes/deephaven_core/proto/config.proto \ /includes/deephaven_core/proto/hierarchicaltable.proto \ /includes/deephaven_core/proto/storage.proto; \ @@ -48,6 +49,7 @@ RUN set -eux; \ /includes/deephaven_core/proto/application.proto \ /includes/deephaven_core/proto/inputtable.proto \ /includes/deephaven_core/proto/partitionedtable.proto \ + /includes/deephaven_core/proto/remotefilesource.proto \ /includes/deephaven_core/proto/config.proto \ /includes/deephaven_core/proto/hierarchicaltable.proto \ /includes/deephaven_core/proto/storage.proto; \ @@ -64,6 +66,7 @@ RUN set -eux; \ /includes/deephaven_core/proto/application.proto \ /includes/deephaven_core/proto/inputtable.proto \ /includes/deephaven_core/proto/partitionedtable.proto \ + /includes/deephaven_core/proto/remotefilesource.proto \ /includes/deephaven_core/proto/config.proto \ /includes/deephaven_core/proto/hierarchicaltable.proto \ /includes/deephaven_core/proto/storage.proto; \ @@ -83,6 +86,7 @@ RUN set -eux; \ /includes/deephaven_core/proto/application.proto \ /includes/deephaven_core/proto/inputtable.proto \ /includes/deephaven_core/proto/partitionedtable.proto \ + /includes/deephaven_core/proto/remotefilesource.proto \ /includes/deephaven_core/proto/config.proto \ /includes/deephaven_core/proto/hierarchicaltable.proto \ /includes/deephaven_core/proto/storage.proto; \ @@ -151,6 +155,12 @@ RUN set -eux; \ --doc_opt=html,partitionedtable.html \ -I/includes \ /includes/deephaven_core/proto/partitionedtable.proto; \ + /opt/protoc/bin/protoc \ + --plugin=protoc-gen-doc=/usr/local/bin/protoc-gen-doc \ + --doc_out=generated/proto-doc/multi-html \ + --doc_opt=html,remotefilesource.html \ + -I/includes \ + /includes/deephaven_core/proto/remotefilesource.proto; \ /opt/protoc/bin/protoc \ --plugin=protoc-gen-doc=/usr/local/bin/protoc-gen-doc \ --doc_out=generated/proto-doc/multi-html \ @@ -218,6 +228,12 @@ RUN set -eux; \ --doc_opt=markdown,partitionedtable.md \ -I/includes \ /includes/deephaven_core/proto/partitionedtable.proto; \ + /opt/protoc/bin/protoc \ + --plugin=protoc-gen-doc=/usr/local/bin/protoc-gen-doc \ + --doc_out=generated/proto-doc/multi-md \ + --doc_opt=markdown,remotefilesource.md \ + -I/includes \ + /includes/deephaven_core/proto/remotefilesource.proto; \ /opt/protoc/bin/protoc \ --plugin=protoc-gen-doc=/usr/local/bin/protoc-gen-doc \ --doc_out=generated/proto-doc/multi-md \ diff --git a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto new file mode 100644 index 00000000000..586630c3c1f --- /dev/null +++ b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending + */ +syntax = "proto3"; + +package io.deephaven.proto.backplane.grpc; + +option java_multiple_files = true; +option optimize_for = SPEED; +option go_package = "github.com/deephaven/deephaven-core/go/internal/proto/remotefilesource"; + +import "deephaven_core/proto/ticket.proto"; + +message RemoteFileSourcePluginRequest { + // A client-specified identifier used to identify the response for this request when multiple requests are in flight. + // This must be set to a unique value. + string request_id = 1; + oneof request { + RemoteFileSourceMetaRequest meta = 2; + RemoteFileSourceSetConnectionIdRequest set_connection_id = 3; + } +} + +// Request meta info from the server plugin +message RemoteFileSourceMetaRequest { +} + +message RemoteFileSourceSetConnectionIdRequest { + // The connection ID to set for the remote file source. + string connection_id = 1; +} + +//// Fetch the remote file source plugin into the specified ticket +//message RemoteFileSourcePluginFetchRequest { +// io.deephaven.proto.backplane.grpc.Ticket result_id = 1; +//} \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index 493387dd5eb..d37ad0eea1c 100644 --- a/settings.gradle +++ b/settings.gradle @@ -318,6 +318,9 @@ project(':plugin-figure').projectDir = file('plugin/figure') include(':plugin-partitionedtable') project(':plugin-partitionedtable').projectDir = file('plugin/partitionedtable') +include(':plugin-remotefilesource') +project(':plugin-remotefilesource').projectDir = file('plugin/remotefilesource') + include(':plugin-hierarchicaltable') project(':plugin-hierarchicaltable').projectDir = file('plugin/hierarchicaltable') From bd985ac49cf25bf42efd3e848b177e017a14df43 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 26 Nov 2025 14:30:52 -0600 Subject: [PATCH 02/57] Command resolver now fetches plugin (#DH-20578) --- .../RemoteFileSourceCommandResolver.java | 99 ++++++++++++++++++- 1 file changed, 97 insertions(+), 2 deletions(-) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java index 669a6130798..d56dd9ca00c 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java @@ -1,10 +1,23 @@ package io.deephaven.remotefilesource; +import com.google.protobuf.Any; +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.rpc.Code; +import io.deephaven.UncheckedDeephavenException; +import io.deephaven.base.verify.Assert; +import io.deephaven.internal.log.LoggerFactory; +import io.deephaven.io.logger.Logger; +import io.deephaven.proto.backplane.grpc.RemoteFileSourcePluginFetchRequest; +import io.deephaven.proto.backplane.grpc.Ticket; +import io.deephaven.proto.util.Exceptions; import io.deephaven.server.session.CommandResolver; import io.deephaven.server.session.SessionState; import io.deephaven.server.session.TicketRouter; import io.deephaven.server.session.WantsTicketRouter; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; import org.apache.arrow.flight.impl.Flight; import org.jetbrains.annotations.Nullable; @@ -12,11 +25,83 @@ import java.util.function.Consumer; public class RemoteFileSourceCommandResolver implements CommandResolver, WantsTicketRouter { + private static final Logger log = LoggerFactory.getLogger(RemoteFileSourceCommandResolver.class); + + private static final String FETCH_PLUGIN_TYPE_URL = + "type.googleapis.com/" + RemoteFileSourcePluginFetchRequest.getDescriptor().getFullName(); + + private static RemoteFileSourcePluginFetchRequest parseFetchRequest(final Any command) { + if (!FETCH_PLUGIN_TYPE_URL.equals(command.getTypeUrl())) { + throw new IllegalArgumentException("Not a valid remotefilesource command: " + command.getTypeUrl()); + } + + final ByteString bytes = command.getValue(); + final RemoteFileSourcePluginFetchRequest request; + try { + request = RemoteFileSourcePluginFetchRequest.parseFrom(bytes); + } catch (InvalidProtocolBufferException e) { + throw new UncheckedDeephavenException("Could not parse RemoteFileSourcePluginFetchRequest", e); + } + return request; + } + + private static Any parseOrNull(final ByteString data) { + try { + return Any.parseFrom(data); + } catch (final InvalidProtocolBufferException e) { + return null; + } + } + + public SessionState.ExportObject fetchPlugin(@Nullable final SessionState session, + final Flight.FlightDescriptor descriptor, + final RemoteFileSourcePluginFetchRequest request) { + final Ticket resultTicket = request.getResultId(); + final boolean hasResultId = !resultTicket.getTicket().isEmpty(); + if (!hasResultId) { + throw new StatusRuntimeException(Status.INVALID_ARGUMENT); + } + + final SessionState.ExportBuilder pluginExportBuilder = + session.newExport(resultTicket, "RemoteFileSourcePluginFetchRequest.resultTicket"); + pluginExportBuilder.require(); + + final SessionState.ExportObject pluginExport = + pluginExportBuilder.submit(RemoteFileSourceServicePlugin::new); + + final Flight.FlightInfo flightInfo = Flight.FlightInfo.newBuilder() + .setFlightDescriptor(descriptor) + .addEndpoint(Flight.FlightEndpoint.newBuilder() + .setTicket(Flight.Ticket.newBuilder() + .setTicket( + resultTicket.getTicket())) + .build()) + .setTotalRecords(-1) + .setTotalBytes(-1) + .build(); + return SessionState.wrapAsExport(flightInfo); + } + @Override public SessionState.ExportObject flightInfoFor(@Nullable final SessionState session, final Flight.FlightDescriptor descriptor, final String logId) { - return null; + if (session == null) { + throw new StatusRuntimeException(Status.UNAUTHENTICATED); + } + + final Any request = parseOrNull(descriptor.getCmd()); + if (request == null) { + log.error().append("Could not parse remotefilesource command.").endl(); + throw Exceptions.statusRuntimeException(Code.INVALID_ARGUMENT, "Could not parse remotefilesource command Any."); + } + + if (FETCH_PLUGIN_TYPE_URL.equals(request.getTypeUrl())) { + return fetchPlugin(session, descriptor, parseFetchRequest(request)); + } + + log.error().append("Invalid pivot command typeUrl: " + request.getTypeUrl()).endl(); + throw Exceptions.statusRuntimeException(Code.INVALID_ARGUMENT, "Invalid typeUrl: " + request.getTypeUrl()); } @Override @@ -32,7 +117,17 @@ public String getLogNameFor(final ByteBuffer ticket, final String logId) { @Override public boolean handlesCommand(final Flight.FlightDescriptor descriptor) { - return false; + // If not CMD, there is an error with io.deephaven.server.session.TicketRouter.getPathResolver / handlesPath + Assert.eq(descriptor.getType(), "descriptor.getType()", Flight.FlightDescriptor.DescriptorType.CMD, "CMD"); + + // No good way to check if this is a valid command without parsing to Any first. + final Any command = parseOrNull(descriptor.getCmd()); + if (command == null) { + return false; + } + + // Check if the command matches any types that this resolver handles. + return FETCH_PLUGIN_TYPE_URL.equals(command.getTypeUrl()); } @Override From 51bb7b53c87eb69cca798fc550a01505482ff447 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 26 Nov 2025 14:34:38 -0600 Subject: [PATCH 03/57] RemoteFileSourceTicketResolverFactoryService (#DH-20578) --- ...emoteFileSourceTicketResolverFactoryService.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceTicketResolverFactoryService.java diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceTicketResolverFactoryService.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceTicketResolverFactoryService.java new file mode 100644 index 00000000000..89fa503b9aa --- /dev/null +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceTicketResolverFactoryService.java @@ -0,0 +1,13 @@ +package io.deephaven.remotefilesource; + +import com.google.auto.service.AutoService; +import io.deephaven.server.runner.TicketResolversFromServiceLoader; +import io.deephaven.server.session.TicketResolver; + +@AutoService(TicketResolversFromServiceLoader.Factory.class) +public class RemoteFileSourceTicketResolverFactoryService implements TicketResolversFromServiceLoader.Factory { + @Override + public TicketResolver create(final TicketResolversFromServiceLoader.TicketResolverOptions options) { + return new RemoteFileSourceCommandResolver(); + } +} From c282d4002c47f9af84432801b260b057ca7dfa19 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 26 Nov 2025 16:39:42 -0600 Subject: [PATCH 04/57] Added stub for JsRemoteFileSourceService (#DH-20578) --- .../deephaven_core/proto/remotefilesource.proto | 8 ++++---- .../remotefilesource/JsRemoteFileSourceService.java | 12 ++++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java diff --git a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto index 586630c3c1f..65c28479286 100644 --- a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto +++ b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto @@ -30,7 +30,7 @@ message RemoteFileSourceSetConnectionIdRequest { string connection_id = 1; } -//// Fetch the remote file source plugin into the specified ticket -//message RemoteFileSourcePluginFetchRequest { -// io.deephaven.proto.backplane.grpc.Ticket result_id = 1; -//} \ No newline at end of file +// Fetch the remote file source plugin into the specified ticket +message RemoteFileSourcePluginFetchRequest { + io.deephaven.proto.backplane.grpc.Ticket result_id = 1; +} \ No newline at end of file diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java new file mode 100644 index 00000000000..d7df99933b0 --- /dev/null +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -0,0 +1,12 @@ +package io.deephaven.web.client.api.remotefilesource; + +import io.deephaven.web.client.api.event.HasEventHandling; +import jsinterop.annotations.JsType; + +/** + * JavaScript client for the RemoteFileSource service. + */ +@JsType(namespace = "dh.remotefilesource", name = "RemoteFileSourceService") +public class JsRemoteFileSourceService extends HasEventHandling { + // TODO: This needs to send a RemoteFileSourcePluginFetchRequest flight message to get a plugin service instance +} From 17357331fdee5325c4061dfab41b58f56729c846 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Wed, 26 Nov 2025 20:16:07 -0600 Subject: [PATCH 05/57] Generated GWT bindings --- .../RemoteFileSourceMetaRequest.java | 30 +++ .../RemoteFileSourcePluginFetchRequest.java | 193 ++++++++++++++++++ .../RemoteFileSourcePluginRequest.java | 143 +++++++++++++ ...emoteFileSourceSetConnectionIdRequest.java | 68 ++++++ .../RequestCase.java | 17 ++ 5 files changed, 451 insertions(+) create mode 100644 web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaRequest.java create mode 100644 web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java create mode 100644 web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginRequest.java create mode 100644 web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceSetConnectionIdRequest.java create mode 100644 web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourcepluginrequest/RequestCase.java diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaRequest.java new file mode 100644 index 00000000000..c2d0cc39470 --- /dev/null +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaRequest.java @@ -0,0 +1,30 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; + +import elemental2.core.Uint8Array; +import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsType; + +@JsType( + isNative = true, + name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceMetaRequest", + namespace = JsPackage.GLOBAL) +public class RemoteFileSourceMetaRequest { + public static native RemoteFileSourceMetaRequest deserializeBinary(Uint8Array bytes); + + public static native RemoteFileSourceMetaRequest deserializeBinaryFromReader( + RemoteFileSourceMetaRequest message, Object reader); + + public static native void serializeBinaryToWriter( + RemoteFileSourceMetaRequest message, Object writer); + + public static native Object toObject(boolean includeInstance, RemoteFileSourceMetaRequest msg); + + public native Uint8Array serializeBinary(); + + public native Object toObject(); + + public native Object toObject(boolean includeInstance); +} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java new file mode 100644 index 00000000000..981a4d40580 --- /dev/null +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java @@ -0,0 +1,193 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; + +import elemental2.core.Uint8Array; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.Ticket; +import jsinterop.annotations.JsOverlay; +import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsProperty; +import jsinterop.annotations.JsType; +import jsinterop.base.Js; +import jsinterop.base.JsPropertyMap; + +@JsType( + isNative = true, + name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourcePluginFetchRequest", + namespace = JsPackage.GLOBAL) +public class RemoteFileSourcePluginFetchRequest { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ResultIdFieldType { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface GetTicketUnionType { + @JsOverlay + static RemoteFileSourcePluginFetchRequest.ToObjectReturnType.ResultIdFieldType.GetTicketUnionType of( + Object o) { + return Js.cast(o); + } + + @JsOverlay + default String asString() { + return Js.asString(this); + } + + @JsOverlay + default Uint8Array asUint8Array() { + return Js.cast(this); + } + + @JsOverlay + default boolean isString() { + return (Object) this instanceof String; + } + + @JsOverlay + default boolean isUint8Array() { + return (Object) this instanceof Uint8Array; + } + } + + @JsOverlay + static RemoteFileSourcePluginFetchRequest.ToObjectReturnType.ResultIdFieldType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + RemoteFileSourcePluginFetchRequest.ToObjectReturnType.ResultIdFieldType.GetTicketUnionType getTicket(); + + @JsProperty + void setTicket( + RemoteFileSourcePluginFetchRequest.ToObjectReturnType.ResultIdFieldType.GetTicketUnionType ticket); + + @JsOverlay + default void setTicket(String ticket) { + setTicket( + Js.uncheckedCast( + ticket)); + } + + @JsOverlay + default void setTicket(Uint8Array ticket) { + setTicket( + Js.uncheckedCast( + ticket)); + } + } + + @JsOverlay + static RemoteFileSourcePluginFetchRequest.ToObjectReturnType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + RemoteFileSourcePluginFetchRequest.ToObjectReturnType.ResultIdFieldType getResultId(); + + @JsProperty + void setResultId( + RemoteFileSourcePluginFetchRequest.ToObjectReturnType.ResultIdFieldType resultId); + } + + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType0 { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ResultIdFieldType { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface GetTicketUnionType { + @JsOverlay + static RemoteFileSourcePluginFetchRequest.ToObjectReturnType0.ResultIdFieldType.GetTicketUnionType of( + Object o) { + return Js.cast(o); + } + + @JsOverlay + default String asString() { + return Js.asString(this); + } + + @JsOverlay + default Uint8Array asUint8Array() { + return Js.cast(this); + } + + @JsOverlay + default boolean isString() { + return (Object) this instanceof String; + } + + @JsOverlay + default boolean isUint8Array() { + return (Object) this instanceof Uint8Array; + } + } + + @JsOverlay + static RemoteFileSourcePluginFetchRequest.ToObjectReturnType0.ResultIdFieldType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + RemoteFileSourcePluginFetchRequest.ToObjectReturnType0.ResultIdFieldType.GetTicketUnionType getTicket(); + + @JsProperty + void setTicket( + RemoteFileSourcePluginFetchRequest.ToObjectReturnType0.ResultIdFieldType.GetTicketUnionType ticket); + + @JsOverlay + default void setTicket(String ticket) { + setTicket( + Js.uncheckedCast( + ticket)); + } + + @JsOverlay + default void setTicket(Uint8Array ticket) { + setTicket( + Js.uncheckedCast( + ticket)); + } + } + + @JsOverlay + static RemoteFileSourcePluginFetchRequest.ToObjectReturnType0 create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + RemoteFileSourcePluginFetchRequest.ToObjectReturnType0.ResultIdFieldType getResultId(); + + @JsProperty + void setResultId( + RemoteFileSourcePluginFetchRequest.ToObjectReturnType0.ResultIdFieldType resultId); + } + + public static native RemoteFileSourcePluginFetchRequest deserializeBinary(Uint8Array bytes); + + public static native RemoteFileSourcePluginFetchRequest deserializeBinaryFromReader( + RemoteFileSourcePluginFetchRequest message, Object reader); + + public static native void serializeBinaryToWriter( + RemoteFileSourcePluginFetchRequest message, Object writer); + + public static native RemoteFileSourcePluginFetchRequest.ToObjectReturnType toObject( + boolean includeInstance, RemoteFileSourcePluginFetchRequest msg); + + public native void clearResultId(); + + public native Ticket getResultId(); + + public native boolean hasResultId(); + + public native Uint8Array serializeBinary(); + + public native void setResultId(); + + public native void setResultId(Ticket value); + + public native RemoteFileSourcePluginFetchRequest.ToObjectReturnType0 toObject(); + + public native RemoteFileSourcePluginFetchRequest.ToObjectReturnType0 toObject( + boolean includeInstance); +} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginRequest.java new file mode 100644 index 00000000000..1acf26d56de --- /dev/null +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginRequest.java @@ -0,0 +1,143 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; + +import elemental2.core.Uint8Array; +import jsinterop.annotations.JsOverlay; +import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsProperty; +import jsinterop.annotations.JsType; +import jsinterop.base.Js; +import jsinterop.base.JsPropertyMap; + +@JsType( + isNative = true, + name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourcePluginRequest", + namespace = JsPackage.GLOBAL) +public class RemoteFileSourcePluginRequest { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface SetConnectionIdFieldType { + @JsOverlay + static RemoteFileSourcePluginRequest.ToObjectReturnType.SetConnectionIdFieldType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + String getConnectionId(); + + @JsProperty + void setConnectionId(String connectionId); + } + + @JsOverlay + static RemoteFileSourcePluginRequest.ToObjectReturnType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + Object getMeta(); + + @JsProperty + String getRequestId(); + + @JsProperty + RemoteFileSourcePluginRequest.ToObjectReturnType.SetConnectionIdFieldType getSetConnectionId(); + + @JsProperty + void setMeta(Object meta); + + @JsProperty + void setRequestId(String requestId); + + @JsProperty + void setSetConnectionId( + RemoteFileSourcePluginRequest.ToObjectReturnType.SetConnectionIdFieldType setConnectionId); + } + + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType0 { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface SetConnectionIdFieldType { + @JsOverlay + static RemoteFileSourcePluginRequest.ToObjectReturnType0.SetConnectionIdFieldType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + String getConnectionId(); + + @JsProperty + void setConnectionId(String connectionId); + } + + @JsOverlay + static RemoteFileSourcePluginRequest.ToObjectReturnType0 create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + Object getMeta(); + + @JsProperty + String getRequestId(); + + @JsProperty + RemoteFileSourcePluginRequest.ToObjectReturnType0.SetConnectionIdFieldType getSetConnectionId(); + + @JsProperty + void setMeta(Object meta); + + @JsProperty + void setRequestId(String requestId); + + @JsProperty + void setSetConnectionId( + RemoteFileSourcePluginRequest.ToObjectReturnType0.SetConnectionIdFieldType setConnectionId); + } + + public static native RemoteFileSourcePluginRequest deserializeBinary(Uint8Array bytes); + + public static native RemoteFileSourcePluginRequest deserializeBinaryFromReader( + RemoteFileSourcePluginRequest message, Object reader); + + public static native void serializeBinaryToWriter( + RemoteFileSourcePluginRequest message, Object writer); + + public static native RemoteFileSourcePluginRequest.ToObjectReturnType toObject( + boolean includeInstance, RemoteFileSourcePluginRequest msg); + + public native void clearMeta(); + + public native void clearSetConnectionId(); + + public native RemoteFileSourceMetaRequest getMeta(); + + public native int getRequestCase(); + + public native String getRequestId(); + + public native RemoteFileSourceSetConnectionIdRequest getSetConnectionId(); + + public native boolean hasMeta(); + + public native boolean hasSetConnectionId(); + + public native Uint8Array serializeBinary(); + + public native void setMeta(); + + public native void setMeta(RemoteFileSourceMetaRequest value); + + public native void setRequestId(String value); + + public native void setSetConnectionId(); + + public native void setSetConnectionId(RemoteFileSourceSetConnectionIdRequest value); + + public native RemoteFileSourcePluginRequest.ToObjectReturnType0 toObject(); + + public native RemoteFileSourcePluginRequest.ToObjectReturnType0 toObject(boolean includeInstance); +} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceSetConnectionIdRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceSetConnectionIdRequest.java new file mode 100644 index 00000000000..590ba5ee643 --- /dev/null +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceSetConnectionIdRequest.java @@ -0,0 +1,68 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; + +import elemental2.core.Uint8Array; +import jsinterop.annotations.JsOverlay; +import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsProperty; +import jsinterop.annotations.JsType; +import jsinterop.base.Js; +import jsinterop.base.JsPropertyMap; + +@JsType( + isNative = true, + name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceSetConnectionIdRequest", + namespace = JsPackage.GLOBAL) +public class RemoteFileSourceSetConnectionIdRequest { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType { + @JsOverlay + static RemoteFileSourceSetConnectionIdRequest.ToObjectReturnType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + String getConnectionId(); + + @JsProperty + void setConnectionId(String connectionId); + } + + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType0 { + @JsOverlay + static RemoteFileSourceSetConnectionIdRequest.ToObjectReturnType0 create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + String getConnectionId(); + + @JsProperty + void setConnectionId(String connectionId); + } + + public static native RemoteFileSourceSetConnectionIdRequest deserializeBinary(Uint8Array bytes); + + public static native RemoteFileSourceSetConnectionIdRequest deserializeBinaryFromReader( + RemoteFileSourceSetConnectionIdRequest message, Object reader); + + public static native void serializeBinaryToWriter( + RemoteFileSourceSetConnectionIdRequest message, Object writer); + + public static native RemoteFileSourceSetConnectionIdRequest.ToObjectReturnType toObject( + boolean includeInstance, RemoteFileSourceSetConnectionIdRequest msg); + + public native String getConnectionId(); + + public native Uint8Array serializeBinary(); + + public native void setConnectionId(String value); + + public native RemoteFileSourceSetConnectionIdRequest.ToObjectReturnType0 toObject(); + + public native RemoteFileSourceSetConnectionIdRequest.ToObjectReturnType0 toObject( + boolean includeInstance); +} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourcepluginrequest/RequestCase.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourcepluginrequest/RequestCase.java new file mode 100644 index 00000000000..c263a6194cb --- /dev/null +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourcepluginrequest/RequestCase.java @@ -0,0 +1,17 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.remotefilesourcepluginrequest; + +import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsType; + +@JsType( + isNative = true, + name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourcePluginRequest.RequestCase", + namespace = JsPackage.GLOBAL) +public class RequestCase { + public static int META, + REQUEST_NOT_SET, + SET_CONNECTION_ID; +} From c051a01f8f0cdd4c2e50093b0e62239821315167 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 3 Dec 2025 17:58:10 -0600 Subject: [PATCH 06/57] Fetching plugin service working (#DH-20578) --- .../RemoteFileSourceCommandResolver.java | 1 + proto/raw-js-openapi/src/index.js | 2 + .../src/shim/remotefilesource_pb.js | 2 + proto/raw-js-openapi/webpack.config.js | 2 +- server/jetty-app-11/build.gradle | 1 + server/jetty-app/build.gradle | 1 + .../deephaven/web/client/api/CoreClient.java | 5 + .../JsRemoteFileSourceService.java | 143 +++++++++++++++++- 8 files changed, 155 insertions(+), 2 deletions(-) create mode 100644 proto/raw-js-openapi/src/shim/remotefilesource_pb.js diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java index d56dd9ca00c..ac2e7b0ba72 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java @@ -30,6 +30,7 @@ public class RemoteFileSourceCommandResolver implements CommandResolver, WantsTi private static final String FETCH_PLUGIN_TYPE_URL = "type.googleapis.com/" + RemoteFileSourcePluginFetchRequest.getDescriptor().getFullName(); + private static RemoteFileSourcePluginFetchRequest parseFetchRequest(final Any command) { if (!FETCH_PLUGIN_TYPE_URL.equals(command.getTypeUrl())) { throw new IllegalArgumentException("Not a valid remotefilesource command: " + command.getTypeUrl()); diff --git a/proto/raw-js-openapi/src/index.js b/proto/raw-js-openapi/src/index.js index 6d7b2e69ff5..9176663e2b6 100644 --- a/proto/raw-js-openapi/src/index.js +++ b/proto/raw-js-openapi/src/index.js @@ -6,6 +6,7 @@ var application_pb = require("deephaven_core/proto/application_pb"); var inputtable_pb = require("deephaven_core/proto/inputtable_pb"); var object_pb = require("deephaven_core/proto/object_pb"); var partitionedtable_pb = require("deephaven_core/proto/partitionedtable_pb"); +var remotefilesource_pb = require("deephaven_core/proto/remotefilesource_pb"); var storage_pb = require("deephaven_core/proto/storage_pb"); var config_pb = require("deephaven_core/proto/config_pb"); var hierarchicaltable_pb = require("deephaven_core/proto/hierarchicaltable_pb"); @@ -46,6 +47,7 @@ var io = { deephaven_core: { object_pb_service, partitionedtable_pb, partitionedtable_pb_service, + remotefilesource_pb, storage_pb, storage_pb_service, config_pb, diff --git a/proto/raw-js-openapi/src/shim/remotefilesource_pb.js b/proto/raw-js-openapi/src/shim/remotefilesource_pb.js new file mode 100644 index 00000000000..01f90ee66e7 --- /dev/null +++ b/proto/raw-js-openapi/src/shim/remotefilesource_pb.js @@ -0,0 +1,2 @@ +Object.assign(exports, require('real/remotefilesource_pb').io.deephaven.proto.backplane.grpc) + diff --git a/proto/raw-js-openapi/webpack.config.js b/proto/raw-js-openapi/webpack.config.js index f0e47f9cbee..8d64a5ac4bd 100644 --- a/proto/raw-js-openapi/webpack.config.js +++ b/proto/raw-js-openapi/webpack.config.js @@ -3,7 +3,7 @@ const path = require('path'); // Workaround for broken codegen from protoc-gen-js using import_style=commonjs_strict, both in // the grpc-web protoc-gen-ts plugin, and in protoc-gen-js itself: const aliases = {}; -for (const proto of ['application', 'config', 'console', 'hierarchicaltable', 'inputtable', 'object', 'partitionedtable', 'session', 'storage', 'table', 'ticket']) { +for (const proto of ['application', 'config', 'console', 'hierarchicaltable', 'inputtable', 'object', 'partitionedtable', 'remotefilesource', 'session', 'storage', 'table', 'ticket']) { // Allows a reference to the real proto files, to be made from the shim aliases[`real/${proto}_pb`] = `${__dirname}/build/js-src/deephaven_core/proto/${proto}_pb`; diff --git a/server/jetty-app-11/build.gradle b/server/jetty-app-11/build.gradle index e075c08a8fa..3a4680c88ca 100644 --- a/server/jetty-app-11/build.gradle +++ b/server/jetty-app-11/build.gradle @@ -12,6 +12,7 @@ dependencies { implementation project(':server-jetty-11') implementation project(':extensions-flight-sql') + implementation project(':plugin-remotefilesource') implementation libs.dagger annotationProcessor libs.dagger.compiler diff --git a/server/jetty-app/build.gradle b/server/jetty-app/build.gradle index 50c7406be2b..728cddc629a 100644 --- a/server/jetty-app/build.gradle +++ b/server/jetty-app/build.gradle @@ -12,6 +12,7 @@ dependencies { implementation project(':server-jetty') implementation project(':extensions-flight-sql') + implementation project(':plugin-remotefilesource') implementation libs.dagger annotationProcessor libs.dagger.compiler diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java b/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java index d1c9ddce944..f5006e186e2 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java @@ -15,6 +15,7 @@ import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.config_pb_service.ConfigServiceClient; import io.deephaven.javascript.proto.dhinternal.jspb.Map; import io.deephaven.web.client.api.event.HasEventHandling; +import io.deephaven.web.client.api.remotefilesource.JsRemoteFileSourceService; import io.deephaven.web.client.api.storage.JsStorageService; import io.deephaven.web.client.fu.JsLog; import io.deephaven.web.client.fu.LazyPromise; @@ -154,6 +155,10 @@ public JsStorageService getStorageService() { return new JsStorageService(ideConnection.connection.get()); } + public JsRemoteFileSourceService getRemoteFileSourceService() { + return new JsRemoteFileSourceService(ideConnection.connection.get()); + } + public Promise getAsIdeConnection() { return Promise.resolve(ideConnection); } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index d7df99933b0..e2c0f843083 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -1,6 +1,19 @@ package io.deephaven.web.client.api.remotefilesource; +import elemental2.core.Uint8Array; +import elemental2.promise.Promise; +import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.FlightDescriptor; +import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.FlightInfo; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourcePluginFetchRequest; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.Ticket; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.TypedTicket; +import io.deephaven.web.client.api.Callbacks; +import io.deephaven.web.client.api.ServerObject; +import io.deephaven.web.client.api.WorkerConnection; import io.deephaven.web.client.api.event.HasEventHandling; +import io.deephaven.web.client.api.widget.JsWidgetExportedObject; +import jsinterop.annotations.JsIgnore; +import jsinterop.annotations.JsMethod; import jsinterop.annotations.JsType; /** @@ -8,5 +21,133 @@ */ @JsType(namespace = "dh.remotefilesource", name = "RemoteFileSourceService") public class JsRemoteFileSourceService extends HasEventHandling { - // TODO: This needs to send a RemoteFileSourcePluginFetchRequest flight message to get a plugin service instance + private final WorkerConnection connection; + + @JsIgnore + public JsRemoteFileSourceService(WorkerConnection connection) { + this.connection = connection; + } + + /** + * Fetches a RemoteFileSource plugin instance from the server. + * + * @return a promise that resolves to a ServerObject representing the RemoteFileSource plugin instance + */ + @JsMethod + public Promise fetchPlugin() { + // Create a new export ticket for the result + Ticket resultTicket = connection.getTickets().newExportTicket(); + + // Create the fetch request + RemoteFileSourcePluginFetchRequest fetchRequest = new RemoteFileSourcePluginFetchRequest(); + fetchRequest.setResultId(resultTicket); + + // Serialize the request to bytes + Uint8Array innerRequestBytes = fetchRequest.serializeBinary(); + + // Wrap in google.protobuf.Any with the proper typeUrl + Uint8Array anyWrappedBytes = wrapInAny( + "type.googleapis.com/io.deephaven.proto.backplane.grpc.RemoteFileSourcePluginFetchRequest", + innerRequestBytes + ); + + // Create a FlightDescriptor with the command + FlightDescriptor descriptor = new FlightDescriptor(); + descriptor.setType(FlightDescriptor.DescriptorType.getCMD()); + descriptor.setCmd(anyWrappedBytes); + + // Send the getFlightInfo request + return Callbacks.grpcUnaryPromise(c -> + connection.flightServiceClient().getFlightInfo(descriptor, connection.metadata(), c::apply) + ).then(flightInfo -> { + // The first endpoint should contain the ticket for the plugin instance + if (flightInfo.getEndpointList().length > 0) { + // Get the Arrow Flight ticket from the endpoint + io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.Ticket flightTicket = + flightInfo.getEndpointList().getAt(0).getTicket(); + + // Convert the Arrow Flight ticket to a Deephaven ticket + Ticket dhTicket = new Ticket(); + dhTicket.setTicket(flightTicket.getTicket_asU8()); + + // Create a TypedTicket for the plugin instance + TypedTicket typedTicket = new TypedTicket(); + typedTicket.setTicket(dhTicket); + typedTicket.setType("RemoteFileSourceService"); + + // Return a ServerObject wrapper + return Promise.resolve(new JsWidgetExportedObject(connection, typedTicket)); + } else { + return Promise.reject("No endpoints returned from RemoteFileSource plugin fetch"); + } + }); + } + + /** + * Wraps a protobuf message in a google.protobuf.Any message. + * + * @param typeUrl the type URL for the message (e.g., "type.googleapis.com/package.MessageName") + * @param messageBytes the serialized protobuf message bytes + * @return the serialized Any message containing the wrapped message + */ + private static Uint8Array wrapInAny(String typeUrl, Uint8Array messageBytes) { + // Encode the type_url string to UTF-8 bytes + Uint8Array typeUrlBytes = stringToUtf8(typeUrl); + + // Calculate sizes for protobuf encoding + // Field 1 (type_url): tag + length + data + int typeUrlTag = (1 << 3) | 2; // field 1, wire type 2 (length-delimited) + int typeUrlFieldSize = sizeOfVarint(typeUrlTag) + sizeOfVarint(typeUrlBytes.length) + typeUrlBytes.length; + + // Field 2 (value): tag + length + data + int valueTag = (2 << 3) | 2; // field 2, wire type 2 (length-delimited) + int valueFieldSize = sizeOfVarint(valueTag) + sizeOfVarint(messageBytes.length) + messageBytes.length; + + int totalSize = typeUrlFieldSize + valueFieldSize; + Uint8Array result = new Uint8Array(totalSize); + int pos = 0; + + // Write type_url field + pos = writeVarint(result, pos, typeUrlTag); + pos = writeVarint(result, pos, typeUrlBytes.length); + for (int i = 0; i < typeUrlBytes.length; i++) { + result.setAt(pos++, typeUrlBytes.getAt(i)); + } + + // Write value field + pos = writeVarint(result, pos, valueTag); + pos = writeVarint(result, pos, messageBytes.length); + for (int i = 0; i < messageBytes.length; i++) { + result.setAt(pos++, messageBytes.getAt(i)); + } + + return result; + } + + private static Uint8Array stringToUtf8(String str) { + // Simple UTF-8 encoding for ASCII-compatible strings + Uint8Array bytes = new Uint8Array(str.length()); + for (int i = 0; i < str.length(); i++) { + bytes.setAt(i, (double) str.charAt(i)); + } + return bytes; + } + + private static int sizeOfVarint(int value) { + if (value < 0) return 10; + if (value < 128) return 1; + if (value < 16384) return 2; + if (value < 2097152) return 3; + if (value < 268435456) return 4; + return 5; + } + + private static int writeVarint(Uint8Array buffer, int pos, int value) { + while (value >= 128) { + buffer.setAt(pos++, (double) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + buffer.setAt(pos++, (double) value); + return pos; + } } From ae7d97ec8376d7ec0311fbf6e79d14ea237a347b Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Thu, 4 Dec 2025 09:33:35 -0600 Subject: [PATCH 07/57] Wiring up messagestream (#DH-20578) --- .../deephaven/web/client/api/CoreClient.java | 4 +- .../JsRemoteFileSourceService.java | 148 ++++++++++++++++-- 2 files changed, 139 insertions(+), 13 deletions(-) diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java b/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java index f5006e186e2..ecdeee17289 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java @@ -155,8 +155,8 @@ public JsStorageService getStorageService() { return new JsStorageService(ideConnection.connection.get()); } - public JsRemoteFileSourceService getRemoteFileSourceService() { - return new JsRemoteFileSourceService(ideConnection.connection.get()); + public Promise getRemoteFileSourceService() { + return JsRemoteFileSourceService.fetchPlugin(ideConnection.connection.get()); } public Promise getAsIdeConnection() { diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index e2c0f843083..5de42dfd637 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -1,40 +1,72 @@ package io.deephaven.web.client.api.remotefilesource; +import com.vertispan.tsdefs.annotations.TsInterface; +import com.vertispan.tsdefs.annotations.TsName; import elemental2.core.Uint8Array; +import elemental2.dom.DomGlobal; import elemental2.promise.Promise; import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.FlightDescriptor; import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.FlightInfo; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.object_pb.ConnectRequest; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.object_pb.StreamRequest; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.object_pb.StreamResponse; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourcePluginFetchRequest; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.Ticket; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.TypedTicket; import io.deephaven.web.client.api.Callbacks; -import io.deephaven.web.client.api.ServerObject; import io.deephaven.web.client.api.WorkerConnection; +import io.deephaven.web.client.api.barrage.stream.BiDiStream; import io.deephaven.web.client.api.event.HasEventHandling; -import io.deephaven.web.client.api.widget.JsWidgetExportedObject; import jsinterop.annotations.JsIgnore; import jsinterop.annotations.JsMethod; -import jsinterop.annotations.JsType; +import jsinterop.annotations.JsProperty; +import jsinterop.base.Js; + +import java.util.function.Supplier; /** - * JavaScript client for the RemoteFileSource service. + * JavaScript client for the RemoteFileSource service. Provides bidirectional communication with the server-side + * RemoteFileSourceServicePlugin via a message stream. */ -@JsType(namespace = "dh.remotefilesource", name = "RemoteFileSourceService") +@TsInterface +@TsName(namespace = "dh.remotefilesource", name = "RemoteFileSourceService") public class JsRemoteFileSourceService extends HasEventHandling { + @JsProperty(namespace = "dh.remotefilesource.RemoteFileSourceService") + public static final String EVENT_MESSAGE = "message"; + @JsProperty(namespace = "dh.remotefilesource.RemoteFileSourceService") + public static final String EVENT_CLOSE = "close"; + private final WorkerConnection connection; + private final TypedTicket typedTicket; + + private final Supplier> streamFactory; + private BiDiStream messageStream; + + private boolean hasFetched; @JsIgnore - public JsRemoteFileSourceService(WorkerConnection connection) { + private JsRemoteFileSourceService(WorkerConnection connection, TypedTicket typedTicket) { this.connection = connection; + this.typedTicket = typedTicket; + this.hasFetched = false; + + // Set up the message stream factory + BiDiStream.Factory factory = connection.streamFactory(); + this.streamFactory = () -> factory.create( + connection.objectServiceClient()::messageStream, + (first, headers) -> connection.objectServiceClient().openMessageStream(first, headers), + (next, headers, c) -> connection.objectServiceClient().nextMessageStream(next, headers, c::apply), + new StreamRequest()); } /** - * Fetches a RemoteFileSource plugin instance from the server. + * Fetches a RemoteFileSource plugin instance from the server and establishes a message stream connection. * - * @return a promise that resolves to a ServerObject representing the RemoteFileSource plugin instance + * @param connection the worker connection to use for communication + * @return a promise that resolves to a RemoteFileSourceService instance with an active message stream */ @JsMethod - public Promise fetchPlugin() { + public static Promise fetchPlugin(WorkerConnection connection) { // Create a new export ticket for the result Ticket resultTicket = connection.getTickets().newExportTicket(); @@ -75,14 +107,107 @@ public Promise fetchPlugin() { typedTicket.setTicket(dhTicket); typedTicket.setType("RemoteFileSourceService"); - // Return a ServerObject wrapper - return Promise.resolve(new JsWidgetExportedObject(connection, typedTicket)); + // Create a new service instance with the typed ticket and connect to it + JsRemoteFileSourceService service = new JsRemoteFileSourceService(connection, typedTicket); + return service.connect(); } else { return Promise.reject("No endpoints returned from RemoteFileSource plugin fetch"); } }); } + /** + * Establishes the message stream connection to the server-side plugin instance. + * + * @return a promise that resolves to this service instance when the connection is established + */ + @JsIgnore + private Promise connect() { + if (messageStream != null) { + messageStream.end(); + } + + return new Promise<>((resolve, reject) -> { + messageStream = streamFactory.get(); + + messageStream.onData(res -> { + if (!hasFetched) { + hasFetched = true; + resolve.onInvoke(this); + } else { + // Fire message event for subsequent messages + DomGlobal.setTimeout(ignore -> { + fireEvent(EVENT_MESSAGE, res.getData()); + }, 0); + } + }); + + messageStream.onStatus(status -> { + if (!status.isOk()) { + reject.onInvoke(status.getDetails()); + } + DomGlobal.setTimeout(ignore -> { + fireEvent(EVENT_CLOSE); + }, 0); + closeStream(); + }); + + messageStream.onEnd(status -> { + closeStream(); + }); + + // First message establishes a connection w/ the plugin object instance we're talking to + StreamRequest req = new StreamRequest(); + ConnectRequest data = new ConnectRequest(); + data.setSourceId(typedTicket); + req.setConnect(data); + messageStream.send(req); + }); + } + + /** + * Sends a message to the server-side plugin. + * + * @param payload the message data to send (string, ArrayBuffer, or typed array) + */ + @JsMethod + public void sendMessage(Object payload) { + if (messageStream == null) { + throw new IllegalStateException("Message stream not connected"); + } + + StreamRequest req = new StreamRequest(); + + // Convert the payload to Uint8Array + Uint8Array data; + if (payload instanceof String) { + data = stringToUtf8((String) payload); + } else if (payload instanceof Uint8Array) { + data = (Uint8Array) payload; + } else { + data = Js.uncheckedCast(payload); + } + + req.getData().setPayload(data); + messageStream.send(req); + } + + /** + * Closes the message stream connection to the server. + */ + @JsMethod + public void close() { + closeStream(); + } + + @JsIgnore + private void closeStream() { + if (messageStream != null) { + messageStream.end(); + messageStream = null; + } + } + /** * Wraps a protobuf message in a google.protobuf.Any message. * @@ -151,3 +276,4 @@ private static int writeVarint(Uint8Array buffer, int pos, int value) { return pos; } } + From ef53eeb2e6fd7c683b124b6245db5ebb3fa8b36b Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Thu, 4 Dec 2025 11:15:00 -0600 Subject: [PATCH 08/57] test bidirectional communication (#DH-20578) --- .../RemoteFileSourceServicePlugin.java | 161 ++++++++++++++++-- .../proto/remotefilesource.proto | 44 ++++- .../JsRemoteFileSourceService.java | 138 ++++++++++++++- .../RemoteFileSourceClientRequest.java | 59 +++++++ .../RemoteFileSourceMetaRequest.java | 8 +- .../RemoteFileSourceMetaResponse.java | 43 +++++ .../RemoteFileSourcePluginRequest.java | 143 ---------------- .../RemoteFileSourceServerRequest.java | 41 +++++ 8 files changed, 469 insertions(+), 168 deletions(-) create mode 100644 web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java create mode 100644 web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaResponse.java delete mode 100644 web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginRequest.java create mode 100644 web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java index 9fcf6bc8505..df99feeca4f 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java @@ -2,15 +2,30 @@ import com.google.auto.service.AutoService; import com.google.protobuf.InvalidProtocolBufferException; +import io.deephaven.internal.log.LoggerFactory; +import io.deephaven.io.logger.Logger; import io.deephaven.plugin.type.ObjectType; import io.deephaven.plugin.type.ObjectTypeBase; import io.deephaven.plugin.type.ObjectCommunicationException; -import io.deephaven.proto.backplane.grpc.RemoteFileSourcePluginFetchRequest; +import io.deephaven.proto.backplane.grpc.RemoteFileSourceClientRequest; +import io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaRequest; +import io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaResponse; +import io.deephaven.proto.backplane.grpc.RemoteFileSourceServerRequest; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; @AutoService(ObjectType.class) public class RemoteFileSourceServicePlugin extends ObjectTypeBase { + private static final Logger log = LoggerFactory.getLogger(RemoteFileSourceServicePlugin.class); + + private volatile RemoteFileSourceMessageStream messageStream; + public RemoteFileSourceServicePlugin() {} @Override @@ -26,14 +41,36 @@ public boolean isType(Object object) { @Override public MessageStream compatibleClientConnection(Object object, MessageStream connection) throws ObjectCommunicationException { connection.onData(ByteBuffer.allocate(0)); - return new RemoteFileSourceMessageStream(connection); + messageStream = new RemoteFileSourceMessageStream(connection); + return messageStream; + } + + /** + * Test method to trigger a resource request from the server to the client. + * Can be called from the console to test bidirectional communication. + * + * Usage from console: + * 
+     * service = remote_file_source_service  # The plugin instance
+     * service.testRequestResource("com/example/MyClass.java")
+     *
+ * + * @param resourceName the resource to request from the client + */ + public void testRequestResource(String resourceName) { + if (messageStream == null) { + log.error().append("MessageStream not connected. Please connect a client first.").endl(); + return; + } + messageStream.testRequestResource(resourceName); } /** * A message stream for the RemoteFileSourceService. */ - private class RemoteFileSourceMessageStream implements MessageStream { + private static class RemoteFileSourceMessageStream implements MessageStream { private final MessageStream connection; + private final Map> pendingRequests = new ConcurrentHashMap<>(); public RemoteFileSourceMessageStream(final MessageStream connection) { this.connection = connection; @@ -41,21 +78,125 @@ public RemoteFileSourceMessageStream(final MessageStream connection) { @Override public void onData(ByteBuffer payload, Object... references) throws ObjectCommunicationException { - final RemoteFileSourcePluginFetchRequest request; - try { - request = RemoteFileSourcePluginFetchRequest.parseFrom(payload); + // Parse as RemoteFileSourceClientRequest proto (client→server) + byte[] bytes = new byte[payload.remaining()]; + payload.get(bytes); + RemoteFileSourceClientRequest message = RemoteFileSourceClientRequest.parseFrom(bytes); + + String requestId = message.getRequestId(); + + if (message.hasMetaResponse()) { + // Client is responding to a resource request + RemoteFileSourceMetaResponse response = message.getMetaResponse(); + + CompletableFuture future = pendingRequests.remove(requestId); + if (future != null) { + byte[] content = response.getContent().toByteArray(); + + log.info().append("Received resource response for requestId: ").append(requestId) + .append(", found: ").append(response.getFound()) + .append(", content length: ").append(content.length).endl(); + + if (!response.getError().isEmpty()) { + log.warn().append("Error in response: ").append(response.getError()).endl(); + } + + // Complete the future - the caller will log the content if needed + future.complete(content); + } else { + log.warn().append("Received response for unknown requestId: ").append(requestId).endl(); + } + } else if (message.hasSetConnectionId()) { + // Client sent connection ID (future use) + log.info().append("Received set_connection_id from client").endl(); + } else if (message.hasTestCommand()) { + // Client sent a test command + String command = message.getTestCommand(); + log.info().append("Received test command from client: ").append(command).endl(); + + if (command.startsWith("TEST:")) { + String resourceName = command.substring(5).trim(); + log.info().append("Client initiated test for resource: ").append(resourceName).endl(); + testRequestResource(resourceName); + } + } else { + log.warn().append("Received unknown message type from client").endl(); + } } catch (InvalidProtocolBufferException e) { - // There is no identifier here, so we cannot properly return an error that is bound to the request. - // Instead, we throw an Exception causing the server to close the entire MessageStream and - // propagate a general error to the client. - throw new RuntimeException(e); + log.error().append("Failed to parse RemoteFileSourceClientRequest: ").append(e).endl(); + throw new ObjectCommunicationException("Failed to parse message", e); } } @Override public void onClose() { + // Cancel all pending requests + pendingRequests.values().forEach(future -> future.cancel(true)); + pendingRequests.clear(); + } + + /** + * Request a resource from the client. + * + * @param resourceName the name/path of the resource to request + * @return a future that completes with the resource content, or empty array if not found + */ + public CompletableFuture requestResource(String resourceName) { + String requestId = UUID.randomUUID().toString(); + CompletableFuture future = new CompletableFuture<>(); + pendingRequests.put(requestId, future); + + try { + // Build RemoteFileSourceMetaRequest proto + RemoteFileSourceMetaRequest metaRequest = RemoteFileSourceMetaRequest.newBuilder() + .setResourceName(resourceName) + .build(); + + // Wrap in RemoteFileSourceServerRequest (server→client) + RemoteFileSourceServerRequest message = RemoteFileSourceServerRequest.newBuilder() + .setRequestId(requestId) + .setMetaRequest(metaRequest) + .build(); + + ByteBuffer buffer = ByteBuffer.wrap(message.toByteArray()); + + log.info().append("Sending resource request for: ").append(resourceName) + .append(" with requestId: ").append(requestId).endl(); + + connection.onData(buffer); + } catch (ObjectCommunicationException e) { + future.completeExceptionally(e); + pendingRequests.remove(requestId); + } + + return future; + } + + /** + * Test method to request a resource and log the result. + * This can be called from the server console to test the bidirectional communication. + * + * @param resourceName the resource to request + */ + public void testRequestResource(String resourceName) { + log.info().append("Testing resource request for: ").append(resourceName).endl(); + requestResource(resourceName) + .orTimeout(30, TimeUnit.SECONDS) + .whenComplete((content, error) -> { + if (error != null) { + log.error().append("Error requesting resource ").append(resourceName) + .append(": ").append(error).endl(); + } else { + log.info().append("Successfully received resource ").append(resourceName) + .append(" (").append(content.length).append(" bytes)").endl(); + if (content.length > 0 && content.length < 1000) { + String contentStr = new String(content, StandardCharsets.UTF_8); + log.info().append("Resource content:\n").append(contentStr).endl(); + } + } + }); } } } diff --git a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto index 65c28479286..5f6aff62a6c 100644 --- a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto +++ b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto @@ -11,18 +11,50 @@ option go_package = "github.com/deephaven/deephaven-core/go/internal/proto/remot import "deephaven_core/proto/ticket.proto"; -message RemoteFileSourcePluginRequest { - // A client-specified identifier used to identify the response for this request when multiple requests are in flight. - // This must be set to a unique value. +// Server → Client: Requests sent from server to client via MessageStream +message RemoteFileSourceServerRequest { + // Unique identifier for this request, used to correlate the response string request_id = 1; + + oneof request { + // Request source data/resource from the client + RemoteFileSourceMetaRequest meta_request = 2; + } +} + +// Client → Server: Requests/responses sent from client to server via MessageStream +message RemoteFileSourceClientRequest { + // The request_id from the ServerRequest this is responding to (if applicable) + string request_id = 1; + oneof request { - RemoteFileSourceMetaRequest meta = 2; + // Response to a resource request + RemoteFileSourceMetaResponse meta_response = 2; + + // Future: other client-initiated messages RemoteFileSourceSetConnectionIdRequest set_connection_id = 3; + + // Test command (e.g., "TEST:com/example/Test.java") - client triggers server to request a resource back + string test_command = 4; } } -// Request meta info from the server plugin +// Request source data/resource from the client message RemoteFileSourceMetaRequest { + // The name/path of the resource being requested (e.g., "com/example/MyClass.java") + string resource_name = 1; +} + +// Response to a resource request +message RemoteFileSourceMetaResponse { + // The content of the resource, or empty if not found + bytes content = 1; + + // Indicates whether the resource was found + bool found = 2; + + // Optional: error message if the resource could not be retrieved + string error = 3; } message RemoteFileSourceSetConnectionIdRequest { @@ -30,7 +62,7 @@ message RemoteFileSourceSetConnectionIdRequest { string connection_id = 1; } -// Fetch the remote file source plugin into the specified ticket +// Fetch the remote file source plugin into the specified ticket (Flight command, not MessageStream) message RemoteFileSourcePluginFetchRequest { io.deephaven.proto.backplane.grpc.Ticket result_id = 1; } \ No newline at end of file diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index 5de42dfd637..1f07343d979 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -7,10 +7,15 @@ import elemental2.promise.Promise; import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.FlightDescriptor; import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.FlightInfo; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.object_pb.ClientData; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.object_pb.ConnectRequest; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.object_pb.StreamRequest; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.object_pb.StreamResponse; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceClientRequest; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceMetaRequest; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceMetaResponse; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourcePluginFetchRequest; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceServerRequest; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.Ticket; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.TypedTicket; import io.deephaven.web.client.api.Callbacks; @@ -35,6 +40,8 @@ public class JsRemoteFileSourceService extends HasEventHandling { public static final String EVENT_MESSAGE = "message"; @JsProperty(namespace = "dh.remotefilesource.RemoteFileSourceService") public static final String EVENT_CLOSE = "close"; + @JsProperty(namespace = "dh.remotefilesource.RemoteFileSourceService") + public static final String EVENT_REQUEST = "request"; private final WorkerConnection connection; private final TypedTicket typedTicket; @@ -135,10 +142,33 @@ private Promise connect() { hasFetched = true; resolve.onInvoke(this); } else { - // Fire message event for subsequent messages - DomGlobal.setTimeout(ignore -> { - fireEvent(EVENT_MESSAGE, res.getData()); - }, 0); + // Parse the message as RemoteFileSourceServerRequest proto (server→client) + Uint8Array payload = res.getData().getPayload_asU8(); + + try { + RemoteFileSourceServerRequest message = RemoteFileSourceServerRequest.deserializeBinary(payload); + + // Check which message type it is + if (message.hasMetaRequest()) { + // Server is requesting a resource from the client + RemoteFileSourceMetaRequest request = message.getMetaRequest(); + + // Fire request event (include request_id from wrapper) + DomGlobal.setTimeout(ignore -> { + fireEvent(EVENT_REQUEST, new ResourceRequestEvent(message.getRequestId(), request)); + }, 0); + } else { + // Unknown message type + DomGlobal.setTimeout(ignore -> { + fireEvent(EVENT_MESSAGE, res.getData()); + }, 0); + } + } catch (Exception e) { + // Failed to parse as proto, fire generic message event + DomGlobal.setTimeout(ignore -> { + fireEvent(EVENT_MESSAGE, res.getData()); + }, 0); + } } }); @@ -177,10 +207,13 @@ public void sendMessage(Object payload) { } StreamRequest req = new StreamRequest(); + ClientData clientData = new ClientData(); // Convert the payload to Uint8Array Uint8Array data; if (payload instanceof String) { + // For now, just convert string to UTF-8 bytes + // In the future, we might want to wrap in a proper proto message data = stringToUtf8((String) payload); } else if (payload instanceof Uint8Array) { data = (Uint8Array) payload; @@ -188,7 +221,33 @@ public void sendMessage(Object payload) { data = Js.uncheckedCast(payload); } - req.getData().setPayload(data); + clientData.setPayload(data); + req.setData(clientData); + messageStream.send(req); + } + + /** + * Test method to verify bidirectional communication. + * Sends a test command to the server, which will request a resource back from the client. + * + * @param resourceName the resource name to use for the test (e.g., "com/example/Test.java") + */ + @JsMethod + public void testBidirectionalCommunication(String resourceName) { + if (messageStream == null) { + throw new IllegalStateException("Message stream not connected"); + } + + // Build RemoteFileSourceClientRequest with test_command + RemoteFileSourceClientRequest clientRequest = new RemoteFileSourceClientRequest(); + clientRequest.setRequestId(""); // Empty request_id for test commands + clientRequest.setTestCommand("TEST:" + resourceName); + + // Send via message stream + StreamRequest req = new StreamRequest(); + ClientData clientData = new ClientData(); + clientData.setPayload(clientRequest.serializeBinary()); + req.setData(clientData); messageStream.send(req); } @@ -208,6 +267,75 @@ private void closeStream() { } } + /** + * Event details for a resource request from the server. + * Wraps the proto RemoteFileSourceMetaRequest and provides a respond() method. + */ + @TsInterface + @TsName(namespace = "dh.remotefilesource", name = "ResourceRequest") + public class ResourceRequestEvent { + private final String requestId; + private final RemoteFileSourceMetaRequest protoRequest; + + @JsIgnore + public ResourceRequestEvent(String requestId, RemoteFileSourceMetaRequest protoRequest) { + this.requestId = requestId; + this.protoRequest = protoRequest; + } + + /** + * @return the name/path of the requested resource + */ + @JsProperty + public String getResourceName() { + return protoRequest.getResourceName(); + } + + /** + * Responds to this resource request with the given content. + * + * @param content the resource content (string, ArrayBuffer, or typed array), or null if not found + */ + @JsMethod + public void respond(Object content) { + if (messageStream == null) { + throw new IllegalStateException("Message stream not connected"); + } + + // Build RemoteFileSourceMetaResponse proto + RemoteFileSourceMetaResponse response = new RemoteFileSourceMetaResponse(); + + if (content == null) { + // Resource not found + response.setFound(false); + response.setContent(new Uint8Array(0)); + } else { + response.setFound(true); + + // Convert content to bytes + if (content instanceof String) { + response.setContent(stringToUtf8((String) content)); + } else if (content instanceof Uint8Array) { + response.setContent((Uint8Array) content); + } else { + response.setContent(Js.uncheckedCast(content)); + } + } + + // Wrap in RemoteFileSourceClientRequest (client→server) + RemoteFileSourceClientRequest clientRequest = new RemoteFileSourceClientRequest(); + clientRequest.setRequestId(requestId); + clientRequest.setMetaResponse(response); + + // Send via message stream + StreamRequest req = new StreamRequest(); + ClientData clientData = new ClientData(); + clientData.setPayload(clientRequest.serializeBinary()); + req.setData(clientData); + messageStream.send(req); + } + } + /** * Wraps a protobuf message in a google.protobuf.Any message. * diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java new file mode 100644 index 00000000000..620652de5da --- /dev/null +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java @@ -0,0 +1,59 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; + +import elemental2.core.Uint8Array; +import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsType; + +@JsType( + isNative = true, + name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceClientRequest", + namespace = JsPackage.GLOBAL) +public class RemoteFileSourceClientRequest { + public static native RemoteFileSourceClientRequest deserializeBinary(Uint8Array bytes); + + public static native RemoteFileSourceClientRequest deserializeBinaryFromReader( + RemoteFileSourceClientRequest message, Object reader); + + public static native void serializeBinaryToWriter( + RemoteFileSourceClientRequest message, Object writer); + + public native void clearRequestId(); + + public native void clearMetaResponse(); + + public native void clearSetConnectionId(); + + public native void clearTestCommand(); + + public native String getRequestId(); + + public native RemoteFileSourceMetaResponse getMetaResponse(); + + public native RemoteFileSourceSetConnectionIdRequest getSetConnectionId(); + + public native String getTestCommand(); + + public native boolean hasMetaResponse(); + + public native boolean hasSetConnectionId(); + + public native boolean hasTestCommand(); + + public native Uint8Array serializeBinary(); + + public native void setRequestId(String value); + + public native void setMetaResponse(); + + public native void setMetaResponse(RemoteFileSourceMetaResponse value); + + public native void setSetConnectionId(); + + public native void setSetConnectionId(RemoteFileSourceSetConnectionIdRequest value); + + public native void setTestCommand(String value); +} + diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaRequest.java index c2d0cc39470..303d1a93cb6 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaRequest.java @@ -20,11 +20,11 @@ public static native RemoteFileSourceMetaRequest deserializeBinaryFromReader( public static native void serializeBinaryToWriter( RemoteFileSourceMetaRequest message, Object writer); - public static native Object toObject(boolean includeInstance, RemoteFileSourceMetaRequest msg); + public native void clearResourceName(); - public native Uint8Array serializeBinary(); + public native String getResourceName(); - public native Object toObject(); + public native Uint8Array serializeBinary(); - public native Object toObject(boolean includeInstance); + public native void setResourceName(String value); } diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaResponse.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaResponse.java new file mode 100644 index 00000000000..157c0fad3fb --- /dev/null +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaResponse.java @@ -0,0 +1,43 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; + +import elemental2.core.Uint8Array; +import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsType; + +@JsType( + isNative = true, + name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceMetaResponse", + namespace = JsPackage.GLOBAL) +public class RemoteFileSourceMetaResponse { + public static native RemoteFileSourceMetaResponse deserializeBinary(Uint8Array bytes); + + public static native RemoteFileSourceMetaResponse deserializeBinaryFromReader( + RemoteFileSourceMetaResponse message, Object reader); + + public static native void serializeBinaryToWriter( + RemoteFileSourceMetaResponse message, Object writer); + + public native void clearContent(); + + public native void clearFound(); + + public native void clearError(); + + public native Uint8Array getContent(); + + public native boolean getFound(); + + public native String getError(); + + public native Uint8Array serializeBinary(); + + public native void setContent(Uint8Array value); + + public native void setFound(boolean value); + + public native void setError(String value); +} + diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginRequest.java deleted file mode 100644 index 1acf26d56de..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginRequest.java +++ /dev/null @@ -1,143 +0,0 @@ -// -// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; - -import elemental2.core.Uint8Array; -import jsinterop.annotations.JsOverlay; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsProperty; -import jsinterop.annotations.JsType; -import jsinterop.base.Js; -import jsinterop.base.JsPropertyMap; - -@JsType( - isNative = true, - name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourcePluginRequest", - namespace = JsPackage.GLOBAL) -public class RemoteFileSourcePluginRequest { - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface ToObjectReturnType { - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface SetConnectionIdFieldType { - @JsOverlay - static RemoteFileSourcePluginRequest.ToObjectReturnType.SetConnectionIdFieldType create() { - return Js.uncheckedCast(JsPropertyMap.of()); - } - - @JsProperty - String getConnectionId(); - - @JsProperty - void setConnectionId(String connectionId); - } - - @JsOverlay - static RemoteFileSourcePluginRequest.ToObjectReturnType create() { - return Js.uncheckedCast(JsPropertyMap.of()); - } - - @JsProperty - Object getMeta(); - - @JsProperty - String getRequestId(); - - @JsProperty - RemoteFileSourcePluginRequest.ToObjectReturnType.SetConnectionIdFieldType getSetConnectionId(); - - @JsProperty - void setMeta(Object meta); - - @JsProperty - void setRequestId(String requestId); - - @JsProperty - void setSetConnectionId( - RemoteFileSourcePluginRequest.ToObjectReturnType.SetConnectionIdFieldType setConnectionId); - } - - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface ToObjectReturnType0 { - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface SetConnectionIdFieldType { - @JsOverlay - static RemoteFileSourcePluginRequest.ToObjectReturnType0.SetConnectionIdFieldType create() { - return Js.uncheckedCast(JsPropertyMap.of()); - } - - @JsProperty - String getConnectionId(); - - @JsProperty - void setConnectionId(String connectionId); - } - - @JsOverlay - static RemoteFileSourcePluginRequest.ToObjectReturnType0 create() { - return Js.uncheckedCast(JsPropertyMap.of()); - } - - @JsProperty - Object getMeta(); - - @JsProperty - String getRequestId(); - - @JsProperty - RemoteFileSourcePluginRequest.ToObjectReturnType0.SetConnectionIdFieldType getSetConnectionId(); - - @JsProperty - void setMeta(Object meta); - - @JsProperty - void setRequestId(String requestId); - - @JsProperty - void setSetConnectionId( - RemoteFileSourcePluginRequest.ToObjectReturnType0.SetConnectionIdFieldType setConnectionId); - } - - public static native RemoteFileSourcePluginRequest deserializeBinary(Uint8Array bytes); - - public static native RemoteFileSourcePluginRequest deserializeBinaryFromReader( - RemoteFileSourcePluginRequest message, Object reader); - - public static native void serializeBinaryToWriter( - RemoteFileSourcePluginRequest message, Object writer); - - public static native RemoteFileSourcePluginRequest.ToObjectReturnType toObject( - boolean includeInstance, RemoteFileSourcePluginRequest msg); - - public native void clearMeta(); - - public native void clearSetConnectionId(); - - public native RemoteFileSourceMetaRequest getMeta(); - - public native int getRequestCase(); - - public native String getRequestId(); - - public native RemoteFileSourceSetConnectionIdRequest getSetConnectionId(); - - public native boolean hasMeta(); - - public native boolean hasSetConnectionId(); - - public native Uint8Array serializeBinary(); - - public native void setMeta(); - - public native void setMeta(RemoteFileSourceMetaRequest value); - - public native void setRequestId(String value); - - public native void setSetConnectionId(); - - public native void setSetConnectionId(RemoteFileSourceSetConnectionIdRequest value); - - public native RemoteFileSourcePluginRequest.ToObjectReturnType0 toObject(); - - public native RemoteFileSourcePluginRequest.ToObjectReturnType0 toObject(boolean includeInstance); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java new file mode 100644 index 00000000000..2df13fcdd17 --- /dev/null +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java @@ -0,0 +1,41 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; + +import elemental2.core.Uint8Array; +import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsType; + +@JsType( + isNative = true, + name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceServerRequest", + namespace = JsPackage.GLOBAL) +public class RemoteFileSourceServerRequest { + public static native RemoteFileSourceServerRequest deserializeBinary(Uint8Array bytes); + + public static native RemoteFileSourceServerRequest deserializeBinaryFromReader( + RemoteFileSourceServerRequest message, Object reader); + + public static native void serializeBinaryToWriter( + RemoteFileSourceServerRequest message, Object writer); + + public native void clearRequestId(); + + public native void clearMetaRequest(); + + public native String getRequestId(); + + public native RemoteFileSourceMetaRequest getMetaRequest(); + + public native boolean hasMetaRequest(); + + public native Uint8Array serializeBinary(); + + public native void setRequestId(String value); + + public native void setMetaRequest(); + + public native void setMetaRequest(RemoteFileSourceMetaRequest value); +} + From d3fe8cd7e8eaaa52fac9c6a184b47bfa45aaef3f Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Thu, 4 Dec 2025 12:12:10 -0600 Subject: [PATCH 09/57] set connection id (#DH-20578) --- .../RemoteFileSourceServicePlugin.java | 32 ++++- .../proto/remotefilesource.proto | 12 ++ .../JsRemoteFileSourceService.java | 123 +++++++++--------- .../RemoteFileSourceServerRequest.java | 10 ++ ...moteFileSourceSetConnectionIdResponse.java | 37 ++++++ 5 files changed, 152 insertions(+), 62 deletions(-) create mode 100644 web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceSetConnectionIdResponse.java diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java index df99feeca4f..d577b37eb41 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java @@ -11,6 +11,7 @@ import io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaRequest; import io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaResponse; import io.deephaven.proto.backplane.grpc.RemoteFileSourceServerRequest; +import io.deephaven.proto.backplane.grpc.RemoteFileSourceSetConnectionIdResponse; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; @@ -71,11 +72,19 @@ public void testRequestResource(String resourceName) { private static class RemoteFileSourceMessageStream implements MessageStream { private final MessageStream connection; private final Map> pendingRequests = new ConcurrentHashMap<>(); + private volatile String connectionId; public RemoteFileSourceMessageStream(final MessageStream connection) { this.connection = connection; } + /** + * @return the connection ID set by the client, or null if not set + */ + public String getConnectionId() { + return connectionId; + } + @Override public void onData(ByteBuffer payload, Object... references) throws ObjectCommunicationException { try { @@ -108,8 +117,27 @@ public void onData(ByteBuffer payload, Object... references) throws ObjectCommun log.warn().append("Received response for unknown requestId: ").append(requestId).endl(); } } else if (message.hasSetConnectionId()) { - // Client sent connection ID (future use) - log.info().append("Received set_connection_id from client").endl(); + // Client sent connection ID + String newConnectionId = message.getSetConnectionId().getConnectionId(); + connectionId = newConnectionId; + log.info().append("Set connection ID from client: ").append(newConnectionId).endl(); + + // Send acknowledgment back to client + RemoteFileSourceSetConnectionIdResponse response = RemoteFileSourceSetConnectionIdResponse.newBuilder() + .setConnectionId(newConnectionId) + .setSuccess(true) + .build(); + + RemoteFileSourceServerRequest serverRequest = RemoteFileSourceServerRequest.newBuilder() + .setRequestId(requestId) + .setSetConnectionIdResponse(response) + .build(); + + try { + connection.onData(ByteBuffer.wrap(serverRequest.toByteArray())); + } catch (ObjectCommunicationException e) { + log.error().append("Failed to send connection ID acknowledgment: ").append(e).endl(); + } } else if (message.hasTestCommand()) { // Client sent a test command String command = message.getTestCommand(); diff --git a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto index 5f6aff62a6c..87ab7a50f3e 100644 --- a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto +++ b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto @@ -19,6 +19,9 @@ message RemoteFileSourceServerRequest { oneof request { // Request source data/resource from the client RemoteFileSourceMetaRequest meta_request = 2; + + // Acknowledgment that connection ID was set + RemoteFileSourceSetConnectionIdResponse set_connection_id_response = 3; } } @@ -62,6 +65,15 @@ message RemoteFileSourceSetConnectionIdRequest { string connection_id = 1; } +// Acknowledgment response for setting connection ID +message RemoteFileSourceSetConnectionIdResponse { + // The connection ID that was set + string connection_id = 1; + + // Whether the operation was successful + bool success = 2; +} + // Fetch the remote file source plugin into the specified ticket (Flight command, not MessageStream) message RemoteFileSourcePluginFetchRequest { io.deephaven.proto.backplane.grpc.Ticket result_id = 1; diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index 1f07343d979..e29b8a1b9a4 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -16,6 +16,8 @@ import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceMetaResponse; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourcePluginFetchRequest; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceServerRequest; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceSetConnectionIdRequest; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceSetConnectionIdResponse; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.Ticket; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.TypedTicket; import io.deephaven.web.client.api.Callbacks; @@ -27,6 +29,8 @@ import jsinterop.annotations.JsProperty; import jsinterop.base.Js; +import java.util.HashMap; +import java.util.Map; import java.util.function.Supplier; /** @@ -43,7 +47,6 @@ public class JsRemoteFileSourceService extends HasEventHandling { @JsProperty(namespace = "dh.remotefilesource.RemoteFileSourceService") public static final String EVENT_REQUEST = "request"; - private final WorkerConnection connection; private final TypedTicket typedTicket; private final Supplier> streamFactory; @@ -51,9 +54,12 @@ public class JsRemoteFileSourceService extends HasEventHandling { private boolean hasFetched; + // Track pending setConnectionId requests + private final Map> pendingSetConnectionIdRequests = new HashMap<>(); + private int requestIdCounter = 0; + @JsIgnore private JsRemoteFileSourceService(WorkerConnection connection, TypedTicket typedTicket) { - this.connection = connection; this.typedTicket = typedTicket; this.hasFetched = false; @@ -154,20 +160,26 @@ private Promise connect() { RemoteFileSourceMetaRequest request = message.getMetaRequest(); // Fire request event (include request_id from wrapper) - DomGlobal.setTimeout(ignore -> { - fireEvent(EVENT_REQUEST, new ResourceRequestEvent(message.getRequestId(), request)); - }, 0); + DomGlobal.setTimeout(ignore -> + fireEvent(EVENT_REQUEST, new ResourceRequestEvent(message.getRequestId(), request)), 0); + } else if (message.hasSetConnectionIdResponse()) { + // Server acknowledged connection ID + String requestId = message.getRequestId(); + Promise.PromiseExecutorCallbackFn.ResolveCallbackFn resolveCallback = + pendingSetConnectionIdRequests.remove(requestId); + if (resolveCallback != null) { + RemoteFileSourceSetConnectionIdResponse response = message.getSetConnectionIdResponse(); + resolveCallback.onInvoke(response.getSuccess()); + } } else { // Unknown message type - DomGlobal.setTimeout(ignore -> { - fireEvent(EVENT_MESSAGE, res.getData()); - }, 0); + DomGlobal.setTimeout(ignore -> + fireEvent(EVENT_MESSAGE, res.getData()), 0); } } catch (Exception e) { // Failed to parse as proto, fire generic message event - DomGlobal.setTimeout(ignore -> { - fireEvent(EVENT_MESSAGE, res.getData()); - }, 0); + DomGlobal.setTimeout(ignore -> + fireEvent(EVENT_MESSAGE, res.getData()), 0); } } }); @@ -176,15 +188,13 @@ private Promise connect() { if (!status.isOk()) { reject.onInvoke(status.getDetails()); } - DomGlobal.setTimeout(ignore -> { - fireEvent(EVENT_CLOSE); - }, 0); + DomGlobal.setTimeout(ignore -> + fireEvent(EVENT_CLOSE), 0); closeStream(); }); - messageStream.onEnd(status -> { - closeStream(); - }); + messageStream.onEnd(status -> + closeStream()); // First message establishes a connection w/ the plugin object instance we're talking to StreamRequest req = new StreamRequest(); @@ -196,54 +206,56 @@ private Promise connect() { } /** - * Sends a message to the server-side plugin. + * Test method to verify bidirectional communication. + * Sends a test command to the server, which will request a resource back from the client. * - * @param payload the message data to send (string, ArrayBuffer, or typed array) + * @param resourceName the resource name to use for the test (e.g., "com/example/Test.java") */ @JsMethod - public void sendMessage(Object payload) { - if (messageStream == null) { - throw new IllegalStateException("Message stream not connected"); - } + public void testBidirectionalCommunication(String resourceName) { + RemoteFileSourceClientRequest clientRequest = new RemoteFileSourceClientRequest(); + clientRequest.setRequestId(""); // Empty request_id for test commands + clientRequest.setTestCommand("TEST:" + resourceName); + sendClientRequest(clientRequest); + } - StreamRequest req = new StreamRequest(); - ClientData clientData = new ClientData(); + /** + * Sets the connection ID for this service instance. + * This allows the server to identify and track this specific client connection. + * + * @param connectionId a unique identifier for this connection + * @return a promise that resolves to true if the server successfully set the connection ID, false otherwise + */ + @JsMethod + public Promise setConnectionId(String connectionId) { + return new Promise<>((resolve, reject) -> { + // Generate a unique request ID + String requestId = "setConnectionId-" + (requestIdCounter++); - // Convert the payload to Uint8Array - Uint8Array data; - if (payload instanceof String) { - // For now, just convert string to UTF-8 bytes - // In the future, we might want to wrap in a proper proto message - data = stringToUtf8((String) payload); - } else if (payload instanceof Uint8Array) { - data = (Uint8Array) payload; - } else { - data = Js.uncheckedCast(payload); - } + // Store the resolve callback to call when we get the acknowledgment + pendingSetConnectionIdRequests.put(requestId, resolve); - clientData.setPayload(data); - req.setData(clientData); - messageStream.send(req); + RemoteFileSourceSetConnectionIdRequest setConnIdRequest = new RemoteFileSourceSetConnectionIdRequest(); + setConnIdRequest.setConnectionId(connectionId); + + RemoteFileSourceClientRequest clientRequest = new RemoteFileSourceClientRequest(); + clientRequest.setRequestId(requestId); + clientRequest.setSetConnectionId(setConnIdRequest); + sendClientRequest(clientRequest); + }); } /** - * Test method to verify bidirectional communication. - * Sends a test command to the server, which will request a resource back from the client. + * Helper method to send a RemoteFileSourceClientRequest to the server. * - * @param resourceName the resource name to use for the test (e.g., "com/example/Test.java") + * @param clientRequest the client request to send */ - @JsMethod - public void testBidirectionalCommunication(String resourceName) { + @JsIgnore + private void sendClientRequest(RemoteFileSourceClientRequest clientRequest) { if (messageStream == null) { throw new IllegalStateException("Message stream not connected"); } - // Build RemoteFileSourceClientRequest with test_command - RemoteFileSourceClientRequest clientRequest = new RemoteFileSourceClientRequest(); - clientRequest.setRequestId(""); // Empty request_id for test commands - clientRequest.setTestCommand("TEST:" + resourceName); - - // Send via message stream StreamRequest req = new StreamRequest(); ClientData clientData = new ClientData(); clientData.setPayload(clientRequest.serializeBinary()); @@ -298,10 +310,6 @@ public String getResourceName() { */ @JsMethod public void respond(Object content) { - if (messageStream == null) { - throw new IllegalStateException("Message stream not connected"); - } - // Build RemoteFileSourceMetaResponse proto RemoteFileSourceMetaResponse response = new RemoteFileSourceMetaResponse(); @@ -327,12 +335,7 @@ public void respond(Object content) { clientRequest.setRequestId(requestId); clientRequest.setMetaResponse(response); - // Send via message stream - StreamRequest req = new StreamRequest(); - ClientData clientData = new ClientData(); - clientData.setPayload(clientRequest.serializeBinary()); - req.setData(clientData); - messageStream.send(req); + sendClientRequest(clientRequest); } } diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java index 2df13fcdd17..837afccec1d 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java @@ -24,12 +24,18 @@ public static native void serializeBinaryToWriter( public native void clearMetaRequest(); + public native void clearSetConnectionIdResponse(); + public native String getRequestId(); public native RemoteFileSourceMetaRequest getMetaRequest(); + public native RemoteFileSourceSetConnectionIdResponse getSetConnectionIdResponse(); + public native boolean hasMetaRequest(); + public native boolean hasSetConnectionIdResponse(); + public native Uint8Array serializeBinary(); public native void setRequestId(String value); @@ -37,5 +43,9 @@ public static native void serializeBinaryToWriter( public native void setMetaRequest(); public native void setMetaRequest(RemoteFileSourceMetaRequest value); + + public native void setSetConnectionIdResponse(); + + public native void setSetConnectionIdResponse(RemoteFileSourceSetConnectionIdResponse value); } diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceSetConnectionIdResponse.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceSetConnectionIdResponse.java new file mode 100644 index 00000000000..82ab25ef314 --- /dev/null +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceSetConnectionIdResponse.java @@ -0,0 +1,37 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; + +import elemental2.core.Uint8Array; +import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsType; + +@JsType( + isNative = true, + name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceSetConnectionIdResponse", + namespace = JsPackage.GLOBAL) +public class RemoteFileSourceSetConnectionIdResponse { + public static native RemoteFileSourceSetConnectionIdResponse deserializeBinary(Uint8Array bytes); + + public static native RemoteFileSourceSetConnectionIdResponse deserializeBinaryFromReader( + RemoteFileSourceSetConnectionIdResponse message, Object reader); + + public static native void serializeBinaryToWriter( + RemoteFileSourceSetConnectionIdResponse message, Object writer); + + public native void clearConnectionId(); + + public native void clearSuccess(); + + public native String getConnectionId(); + + public native boolean getSuccess(); + + public native Uint8Array serializeBinary(); + + public native void setConnectionId(String value); + + public native void setSuccess(boolean value); +} + From f8e6dc4a30260441d58e994b8b26fc95ef63ab81 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Thu, 4 Dec 2025 12:36:04 -0600 Subject: [PATCH 10/57] Moved clientSessionId to plugin fetch instead of separate message (#DH-20578) --- .../RemoteFileSourceCommandResolver.java | 6 +- .../RemoteFileSourceServicePlugin.java | 43 +++++------- .../proto/remotefilesource.proto | 24 ++----- .../deephaven/web/client/api/CoreClient.java | 4 +- .../JsRemoteFileSourceService.java | 49 ++----------- .../RemoteFileSourceClientRequest.java | 9 --- .../RemoteFileSourcePluginFetchRequest.java | 6 ++ .../RemoteFileSourceServerRequest.java | 10 --- ...emoteFileSourceSetConnectionIdRequest.java | 68 ------------------- ...moteFileSourceSetConnectionIdResponse.java | 37 ---------- 10 files changed, 39 insertions(+), 217 deletions(-) delete mode 100644 web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceSetConnectionIdRequest.java delete mode 100644 web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceSetConnectionIdResponse.java diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java index ac2e7b0ba72..bbdd6cdc921 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java @@ -63,12 +63,16 @@ public SessionState.ExportObject fetchPlugin(@Nullable final throw new StatusRuntimeException(Status.INVALID_ARGUMENT); } + // Extract optional client session ID from the request (empty string means not provided) + final String clientSessionId = request.getClientSessionId(); + final SessionState.ExportBuilder pluginExportBuilder = session.newExport(resultTicket, "RemoteFileSourcePluginFetchRequest.resultTicket"); pluginExportBuilder.require(); final SessionState.ExportObject pluginExport = - pluginExportBuilder.submit(RemoteFileSourceServicePlugin::new); + pluginExportBuilder.submit(() -> new RemoteFileSourceServicePlugin( + clientSessionId.isEmpty() ? null : clientSessionId)); final Flight.FlightInfo flightInfo = Flight.FlightInfo.newBuilder() .setFlightDescriptor(descriptor) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java index d577b37eb41..452f1aae77d 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java @@ -11,7 +11,6 @@ import io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaRequest; import io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaResponse; import io.deephaven.proto.backplane.grpc.RemoteFileSourceServerRequest; -import io.deephaven.proto.backplane.grpc.RemoteFileSourceSetConnectionIdResponse; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; @@ -26,8 +25,18 @@ public class RemoteFileSourceServicePlugin extends ObjectTypeBase { private static final Logger log = LoggerFactory.getLogger(RemoteFileSourceServicePlugin.class); private volatile RemoteFileSourceMessageStream messageStream; + private final String clientSessionId; - public RemoteFileSourceServicePlugin() {} + public RemoteFileSourceServicePlugin() { + this(null); + } + + public RemoteFileSourceServicePlugin(String clientSessionId) { + this.clientSessionId = clientSessionId; + if (clientSessionId != null) { + log.info().append("RemoteFileSourceServicePlugin created with clientSessionId: ").append(clientSessionId).endl(); + } + } @Override public String name() { @@ -42,7 +51,7 @@ public boolean isType(Object object) { @Override public MessageStream compatibleClientConnection(Object object, MessageStream connection) throws ObjectCommunicationException { connection.onData(ByteBuffer.allocate(0)); - messageStream = new RemoteFileSourceMessageStream(connection); + messageStream = new RemoteFileSourceMessageStream(connection, clientSessionId); return messageStream; } @@ -74,8 +83,12 @@ private static class RemoteFileSourceMessageStream implements MessageStream { private final Map> pendingRequests = new ConcurrentHashMap<>(); private volatile String connectionId; - public RemoteFileSourceMessageStream(final MessageStream connection) { + public RemoteFileSourceMessageStream(final MessageStream connection, final String clientSessionId) { this.connection = connection; + this.connectionId = clientSessionId; // Initialize with the ID from the fetch request + if (clientSessionId != null) { + log.info().append("RemoteFileSourceMessageStream initialized with clientSessionId: ").append(clientSessionId).endl(); + } } /** @@ -116,28 +129,6 @@ public void onData(ByteBuffer payload, Object... references) throws ObjectCommun } else { log.warn().append("Received response for unknown requestId: ").append(requestId).endl(); } - } else if (message.hasSetConnectionId()) { - // Client sent connection ID - String newConnectionId = message.getSetConnectionId().getConnectionId(); - connectionId = newConnectionId; - log.info().append("Set connection ID from client: ").append(newConnectionId).endl(); - - // Send acknowledgment back to client - RemoteFileSourceSetConnectionIdResponse response = RemoteFileSourceSetConnectionIdResponse.newBuilder() - .setConnectionId(newConnectionId) - .setSuccess(true) - .build(); - - RemoteFileSourceServerRequest serverRequest = RemoteFileSourceServerRequest.newBuilder() - .setRequestId(requestId) - .setSetConnectionIdResponse(response) - .build(); - - try { - connection.onData(ByteBuffer.wrap(serverRequest.toByteArray())); - } catch (ObjectCommunicationException e) { - log.error().append("Failed to send connection ID acknowledgment: ").append(e).endl(); - } } else if (message.hasTestCommand()) { // Client sent a test command String command = message.getTestCommand(); diff --git a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto index 87ab7a50f3e..cdfb3e70092 100644 --- a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto +++ b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto @@ -19,9 +19,6 @@ message RemoteFileSourceServerRequest { oneof request { // Request source data/resource from the client RemoteFileSourceMetaRequest meta_request = 2; - - // Acknowledgment that connection ID was set - RemoteFileSourceSetConnectionIdResponse set_connection_id_response = 3; } } @@ -34,11 +31,8 @@ message RemoteFileSourceClientRequest { // Response to a resource request RemoteFileSourceMetaResponse meta_response = 2; - // Future: other client-initiated messages - RemoteFileSourceSetConnectionIdRequest set_connection_id = 3; - // Test command (e.g., "TEST:com/example/Test.java") - client triggers server to request a resource back - string test_command = 4; + string test_command = 3; } } @@ -60,21 +54,11 @@ message RemoteFileSourceMetaResponse { string error = 3; } -message RemoteFileSourceSetConnectionIdRequest { - // The connection ID to set for the remote file source. - string connection_id = 1; -} - -// Acknowledgment response for setting connection ID -message RemoteFileSourceSetConnectionIdResponse { - // The connection ID that was set - string connection_id = 1; - - // Whether the operation was successful - bool success = 2; -} // Fetch the remote file source plugin into the specified ticket (Flight command, not MessageStream) message RemoteFileSourcePluginFetchRequest { io.deephaven.proto.backplane.grpc.Ticket result_id = 1; + + // Optional client session ID to identify this client connection + string client_session_id = 2; } \ No newline at end of file diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java b/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java index ecdeee17289..27473a00f5f 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java @@ -155,8 +155,8 @@ public JsStorageService getStorageService() { return new JsStorageService(ideConnection.connection.get()); } - public Promise getRemoteFileSourceService() { - return JsRemoteFileSourceService.fetchPlugin(ideConnection.connection.get()); + public Promise getRemoteFileSourceService(@JsOptional String clientSessionId) { + return JsRemoteFileSourceService.fetchPlugin(ideConnection.connection.get(), clientSessionId); } public Promise getAsIdeConnection() { diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index e29b8a1b9a4..f621ed157ee 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -16,8 +16,6 @@ import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceMetaResponse; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourcePluginFetchRequest; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceServerRequest; -import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceSetConnectionIdRequest; -import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceSetConnectionIdResponse; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.Ticket; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.TypedTicket; import io.deephaven.web.client.api.Callbacks; @@ -29,8 +27,6 @@ import jsinterop.annotations.JsProperty; import jsinterop.base.Js; -import java.util.HashMap; -import java.util.Map; import java.util.function.Supplier; /** @@ -54,10 +50,6 @@ public class JsRemoteFileSourceService extends HasEventHandling { private boolean hasFetched; - // Track pending setConnectionId requests - private final Map> pendingSetConnectionIdRequests = new HashMap<>(); - private int requestIdCounter = 0; - @JsIgnore private JsRemoteFileSourceService(WorkerConnection connection, TypedTicket typedTicket) { this.typedTicket = typedTicket; @@ -76,16 +68,20 @@ private JsRemoteFileSourceService(WorkerConnection connection, TypedTicket typed * Fetches a RemoteFileSource plugin instance from the server and establishes a message stream connection. * * @param connection the worker connection to use for communication + * @param clientSessionId optional unique identifier for this client session * @return a promise that resolves to a RemoteFileSourceService instance with an active message stream */ @JsMethod - public static Promise fetchPlugin(WorkerConnection connection) { + public static Promise fetchPlugin(WorkerConnection connection, String clientSessionId) { // Create a new export ticket for the result Ticket resultTicket = connection.getTickets().newExportTicket(); // Create the fetch request RemoteFileSourcePluginFetchRequest fetchRequest = new RemoteFileSourcePluginFetchRequest(); fetchRequest.setResultId(resultTicket); + if (clientSessionId != null && !clientSessionId.isEmpty()) { + fetchRequest.setClientSessionId(clientSessionId); + } // Serialize the request to bytes Uint8Array innerRequestBytes = fetchRequest.serializeBinary(); @@ -162,15 +158,6 @@ private Promise connect() { // Fire request event (include request_id from wrapper) DomGlobal.setTimeout(ignore -> fireEvent(EVENT_REQUEST, new ResourceRequestEvent(message.getRequestId(), request)), 0); - } else if (message.hasSetConnectionIdResponse()) { - // Server acknowledged connection ID - String requestId = message.getRequestId(); - Promise.PromiseExecutorCallbackFn.ResolveCallbackFn resolveCallback = - pendingSetConnectionIdRequests.remove(requestId); - if (resolveCallback != null) { - RemoteFileSourceSetConnectionIdResponse response = message.getSetConnectionIdResponse(); - resolveCallback.onInvoke(response.getSuccess()); - } } else { // Unknown message type DomGlobal.setTimeout(ignore -> @@ -219,32 +206,6 @@ public void testBidirectionalCommunication(String resourceName) { sendClientRequest(clientRequest); } - /** - * Sets the connection ID for this service instance. - * This allows the server to identify and track this specific client connection. - * - * @param connectionId a unique identifier for this connection - * @return a promise that resolves to true if the server successfully set the connection ID, false otherwise - */ - @JsMethod - public Promise setConnectionId(String connectionId) { - return new Promise<>((resolve, reject) -> { - // Generate a unique request ID - String requestId = "setConnectionId-" + (requestIdCounter++); - - // Store the resolve callback to call when we get the acknowledgment - pendingSetConnectionIdRequests.put(requestId, resolve); - - RemoteFileSourceSetConnectionIdRequest setConnIdRequest = new RemoteFileSourceSetConnectionIdRequest(); - setConnIdRequest.setConnectionId(connectionId); - - RemoteFileSourceClientRequest clientRequest = new RemoteFileSourceClientRequest(); - clientRequest.setRequestId(requestId); - clientRequest.setSetConnectionId(setConnIdRequest); - sendClientRequest(clientRequest); - }); - } - /** * Helper method to send a RemoteFileSourceClientRequest to the server. * diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java index 620652de5da..28d6736b152 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java @@ -24,22 +24,16 @@ public static native void serializeBinaryToWriter( public native void clearMetaResponse(); - public native void clearSetConnectionId(); - public native void clearTestCommand(); public native String getRequestId(); public native RemoteFileSourceMetaResponse getMetaResponse(); - public native RemoteFileSourceSetConnectionIdRequest getSetConnectionId(); - public native String getTestCommand(); public native boolean hasMetaResponse(); - public native boolean hasSetConnectionId(); - public native boolean hasTestCommand(); public native Uint8Array serializeBinary(); @@ -50,9 +44,6 @@ public static native void serializeBinaryToWriter( public native void setMetaResponse(RemoteFileSourceMetaResponse value); - public native void setSetConnectionId(); - - public native void setSetConnectionId(RemoteFileSourceSetConnectionIdRequest value); public native void setTestCommand(String value); } diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java index 981a4d40580..203f56eb24e 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java @@ -176,8 +176,12 @@ public static native RemoteFileSourcePluginFetchRequest.ToObjectReturnType toObj public native void clearResultId(); + public native void clearClientSessionId(); + public native Ticket getResultId(); + public native String getClientSessionId(); + public native boolean hasResultId(); public native Uint8Array serializeBinary(); @@ -186,6 +190,8 @@ public static native RemoteFileSourcePluginFetchRequest.ToObjectReturnType toObj public native void setResultId(Ticket value); + public native void setClientSessionId(String value); + public native RemoteFileSourcePluginFetchRequest.ToObjectReturnType0 toObject(); public native RemoteFileSourcePluginFetchRequest.ToObjectReturnType0 toObject( diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java index 837afccec1d..2df13fcdd17 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java @@ -24,18 +24,12 @@ public static native void serializeBinaryToWriter( public native void clearMetaRequest(); - public native void clearSetConnectionIdResponse(); - public native String getRequestId(); public native RemoteFileSourceMetaRequest getMetaRequest(); - public native RemoteFileSourceSetConnectionIdResponse getSetConnectionIdResponse(); - public native boolean hasMetaRequest(); - public native boolean hasSetConnectionIdResponse(); - public native Uint8Array serializeBinary(); public native void setRequestId(String value); @@ -43,9 +37,5 @@ public static native void serializeBinaryToWriter( public native void setMetaRequest(); public native void setMetaRequest(RemoteFileSourceMetaRequest value); - - public native void setSetConnectionIdResponse(); - - public native void setSetConnectionIdResponse(RemoteFileSourceSetConnectionIdResponse value); } diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceSetConnectionIdRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceSetConnectionIdRequest.java deleted file mode 100644 index 590ba5ee643..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceSetConnectionIdRequest.java +++ /dev/null @@ -1,68 +0,0 @@ -// -// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; - -import elemental2.core.Uint8Array; -import jsinterop.annotations.JsOverlay; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsProperty; -import jsinterop.annotations.JsType; -import jsinterop.base.Js; -import jsinterop.base.JsPropertyMap; - -@JsType( - isNative = true, - name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceSetConnectionIdRequest", - namespace = JsPackage.GLOBAL) -public class RemoteFileSourceSetConnectionIdRequest { - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface ToObjectReturnType { - @JsOverlay - static RemoteFileSourceSetConnectionIdRequest.ToObjectReturnType create() { - return Js.uncheckedCast(JsPropertyMap.of()); - } - - @JsProperty - String getConnectionId(); - - @JsProperty - void setConnectionId(String connectionId); - } - - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface ToObjectReturnType0 { - @JsOverlay - static RemoteFileSourceSetConnectionIdRequest.ToObjectReturnType0 create() { - return Js.uncheckedCast(JsPropertyMap.of()); - } - - @JsProperty - String getConnectionId(); - - @JsProperty - void setConnectionId(String connectionId); - } - - public static native RemoteFileSourceSetConnectionIdRequest deserializeBinary(Uint8Array bytes); - - public static native RemoteFileSourceSetConnectionIdRequest deserializeBinaryFromReader( - RemoteFileSourceSetConnectionIdRequest message, Object reader); - - public static native void serializeBinaryToWriter( - RemoteFileSourceSetConnectionIdRequest message, Object writer); - - public static native RemoteFileSourceSetConnectionIdRequest.ToObjectReturnType toObject( - boolean includeInstance, RemoteFileSourceSetConnectionIdRequest msg); - - public native String getConnectionId(); - - public native Uint8Array serializeBinary(); - - public native void setConnectionId(String value); - - public native RemoteFileSourceSetConnectionIdRequest.ToObjectReturnType0 toObject(); - - public native RemoteFileSourceSetConnectionIdRequest.ToObjectReturnType0 toObject( - boolean includeInstance); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceSetConnectionIdResponse.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceSetConnectionIdResponse.java deleted file mode 100644 index 82ab25ef314..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceSetConnectionIdResponse.java +++ /dev/null @@ -1,37 +0,0 @@ -// -// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; - -import elemental2.core.Uint8Array; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceSetConnectionIdResponse", - namespace = JsPackage.GLOBAL) -public class RemoteFileSourceSetConnectionIdResponse { - public static native RemoteFileSourceSetConnectionIdResponse deserializeBinary(Uint8Array bytes); - - public static native RemoteFileSourceSetConnectionIdResponse deserializeBinaryFromReader( - RemoteFileSourceSetConnectionIdResponse message, Object reader); - - public static native void serializeBinaryToWriter( - RemoteFileSourceSetConnectionIdResponse message, Object writer); - - public native void clearConnectionId(); - - public native void clearSuccess(); - - public native String getConnectionId(); - - public native boolean getSuccess(); - - public native Uint8Array serializeBinary(); - - public native void setConnectionId(String value); - - public native void setSuccess(boolean value); -} - From e5d8725bc49dd10d21fd95dad41d3d6195e23a9b Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Thu, 4 Dec 2025 14:51:48 -0600 Subject: [PATCH 11/57] Cleanup (#DH-20578) --- .../RemoteFileSourceCommandResolver.java | 10 +- .../RemoteFileSourceServicePlugin.java | 25 +++-- ...ileSourceTicketResolverFactoryService.java | 3 + .../JsRemoteFileSourceService.java | 100 +++++++++--------- 4 files changed, 76 insertions(+), 62 deletions(-) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java index bbdd6cdc921..12276dc5380 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java @@ -1,3 +1,6 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// package io.deephaven.remotefilesource; import com.google.protobuf.Any; @@ -55,8 +58,8 @@ private static Any parseOrNull(final ByteString data) { } public SessionState.ExportObject fetchPlugin(@Nullable final SessionState session, - final Flight.FlightDescriptor descriptor, - final RemoteFileSourcePluginFetchRequest request) { + final Flight.FlightDescriptor descriptor, + final RemoteFileSourcePluginFetchRequest request) { final Ticket resultTicket = request.getResultId(); final boolean hasResultId = !resultTicket.getTicket().isEmpty(); if (!hasResultId) { @@ -98,7 +101,8 @@ public SessionState.ExportObject flightInfoFor(@Nullable fina final Any request = parseOrNull(descriptor.getCmd()); if (request == null) { log.error().append("Could not parse remotefilesource command.").endl(); - throw Exceptions.statusRuntimeException(Code.INVALID_ARGUMENT, "Could not parse remotefilesource command Any."); + throw Exceptions.statusRuntimeException(Code.INVALID_ARGUMENT, + "Could not parse remotefilesource command Any."); } if (FETCH_PLUGIN_TYPE_URL.equals(request.getTypeUrl())) { diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java index 452f1aae77d..ca971c51d86 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java @@ -1,3 +1,6 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// package io.deephaven.remotefilesource; import com.google.auto.service.AutoService; @@ -34,7 +37,8 @@ public RemoteFileSourceServicePlugin() { public RemoteFileSourceServicePlugin(String clientSessionId) { this.clientSessionId = clientSessionId; if (clientSessionId != null) { - log.info().append("RemoteFileSourceServicePlugin created with clientSessionId: ").append(clientSessionId).endl(); + log.info().append("RemoteFileSourceServicePlugin created with clientSessionId: ").append(clientSessionId) + .endl(); } } @@ -49,22 +53,20 @@ public boolean isType(Object object) { } @Override - public MessageStream compatibleClientConnection(Object object, MessageStream connection) throws ObjectCommunicationException { + public MessageStream compatibleClientConnection(Object object, MessageStream connection) + throws ObjectCommunicationException { connection.onData(ByteBuffer.allocate(0)); messageStream = new RemoteFileSourceMessageStream(connection, clientSessionId); return messageStream; } /** - * Test method to trigger a resource request from the server to the client. - * Can be called from the console to test bidirectional communication. - * - * Usage from console: + * Test method to trigger a resource request from the server to the client. Can be called from the console to test + * bidirectional communication. Usage from console: * 
      * service = remote_file_source_service  # The plugin instance
      * service.testRequestResource("com/example/MyClass.java")
      *
- * * @param resourceName the resource to request from the client */ public void testRequestResource(String resourceName) { @@ -81,13 +83,14 @@ public void testRequestResource(String resourceName) { private static class RemoteFileSourceMessageStream implements MessageStream { private final MessageStream connection; private final Map> pendingRequests = new ConcurrentHashMap<>(); - private volatile String connectionId; + private final String connectionId; public RemoteFileSourceMessageStream(final MessageStream connection, final String clientSessionId) { this.connection = connection; this.connectionId = clientSessionId; // Initialize with the ID from the fetch request if (clientSessionId != null) { - log.info().append("RemoteFileSourceMessageStream initialized with clientSessionId: ").append(clientSessionId).endl(); + log.info().append("RemoteFileSourceMessageStream initialized with clientSessionId: ") + .append(clientSessionId).endl(); } } @@ -193,8 +196,8 @@ public CompletableFuture requestResource(String resourceName) { } /** - * Test method to request a resource and log the result. - * This can be called from the server console to test the bidirectional communication. + * Test method to request a resource and log the result. This can be called from the server console to test the + * bidirectional communication. * * @param resourceName the resource to request */ diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceTicketResolverFactoryService.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceTicketResolverFactoryService.java index 89fa503b9aa..fa398c85d95 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceTicketResolverFactoryService.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceTicketResolverFactoryService.java @@ -1,3 +1,6 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// package io.deephaven.remotefilesource; import com.google.auto.service.AutoService; diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index f621ed157ee..44c48ebce58 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -1,3 +1,6 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// package io.deephaven.web.client.api.remotefilesource; import com.vertispan.tsdefs.annotations.TsInterface; @@ -88,9 +91,8 @@ public static Promise fetchPlugin(WorkerConnection co // Wrap in google.protobuf.Any with the proper typeUrl Uint8Array anyWrappedBytes = wrapInAny( - "type.googleapis.com/io.deephaven.proto.backplane.grpc.RemoteFileSourcePluginFetchRequest", - innerRequestBytes - ); + "type.googleapis.com/io.deephaven.proto.backplane.grpc.RemoteFileSourcePluginFetchRequest", + innerRequestBytes); // Create a FlightDescriptor with the command FlightDescriptor descriptor = new FlightDescriptor(); @@ -98,31 +100,31 @@ public static Promise fetchPlugin(WorkerConnection co descriptor.setCmd(anyWrappedBytes); // Send the getFlightInfo request - return Callbacks.grpcUnaryPromise(c -> - connection.flightServiceClient().getFlightInfo(descriptor, connection.metadata(), c::apply) - ).then(flightInfo -> { - // The first endpoint should contain the ticket for the plugin instance - if (flightInfo.getEndpointList().length > 0) { - // Get the Arrow Flight ticket from the endpoint - io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.Ticket flightTicket = - flightInfo.getEndpointList().getAt(0).getTicket(); - - // Convert the Arrow Flight ticket to a Deephaven ticket - Ticket dhTicket = new Ticket(); - dhTicket.setTicket(flightTicket.getTicket_asU8()); - - // Create a TypedTicket for the plugin instance - TypedTicket typedTicket = new TypedTicket(); - typedTicket.setTicket(dhTicket); - typedTicket.setType("RemoteFileSourceService"); - - // Create a new service instance with the typed ticket and connect to it - JsRemoteFileSourceService service = new JsRemoteFileSourceService(connection, typedTicket); - return service.connect(); - } else { - return Promise.reject("No endpoints returned from RemoteFileSource plugin fetch"); - } - }); + return Callbacks.grpcUnaryPromise( + c -> connection.flightServiceClient().getFlightInfo(descriptor, connection.metadata(), c::apply)) + .then(flightInfo -> { + // The first endpoint should contain the ticket for the plugin instance + if (flightInfo.getEndpointList().length > 0) { + // Get the Arrow Flight ticket from the endpoint + io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.Ticket flightTicket = + flightInfo.getEndpointList().getAt(0).getTicket(); + + // Convert the Arrow Flight ticket to a Deephaven ticket + Ticket dhTicket = new Ticket(); + dhTicket.setTicket(flightTicket.getTicket_asU8()); + + // Create a TypedTicket for the plugin instance + TypedTicket typedTicket = new TypedTicket(); + typedTicket.setTicket(dhTicket); + typedTicket.setType("RemoteFileSourceService"); + + // Create a new service instance with the typed ticket and connect to it + JsRemoteFileSourceService service = new JsRemoteFileSourceService(connection, typedTicket); + return service.connect(); + } else { + return Promise.reject("No endpoints returned from RemoteFileSource plugin fetch"); + } + }); } /** @@ -148,7 +150,8 @@ private Promise connect() { Uint8Array payload = res.getData().getPayload_asU8(); try { - RemoteFileSourceServerRequest message = RemoteFileSourceServerRequest.deserializeBinary(payload); + RemoteFileSourceServerRequest message = + RemoteFileSourceServerRequest.deserializeBinary(payload); // Check which message type it is if (message.hasMetaRequest()) { @@ -156,17 +159,15 @@ private Promise connect() { RemoteFileSourceMetaRequest request = message.getMetaRequest(); // Fire request event (include request_id from wrapper) - DomGlobal.setTimeout(ignore -> - fireEvent(EVENT_REQUEST, new ResourceRequestEvent(message.getRequestId(), request)), 0); + DomGlobal.setTimeout(ignore -> fireEvent(EVENT_REQUEST, + new ResourceRequestEvent(message.getRequestId(), request)), 0); } else { // Unknown message type - DomGlobal.setTimeout(ignore -> - fireEvent(EVENT_MESSAGE, res.getData()), 0); + DomGlobal.setTimeout(ignore -> fireEvent(EVENT_MESSAGE, res.getData()), 0); } } catch (Exception e) { // Failed to parse as proto, fire generic message event - DomGlobal.setTimeout(ignore -> - fireEvent(EVENT_MESSAGE, res.getData()), 0); + DomGlobal.setTimeout(ignore -> fireEvent(EVENT_MESSAGE, res.getData()), 0); } } }); @@ -175,13 +176,11 @@ private Promise connect() { if (!status.isOk()) { reject.onInvoke(status.getDetails()); } - DomGlobal.setTimeout(ignore -> - fireEvent(EVENT_CLOSE), 0); + DomGlobal.setTimeout(ignore -> fireEvent(EVENT_CLOSE), 0); closeStream(); }); - messageStream.onEnd(status -> - closeStream()); + messageStream.onEnd(status -> closeStream()); // First message establishes a connection w/ the plugin object instance we're talking to StreamRequest req = new StreamRequest(); @@ -193,8 +192,8 @@ private Promise connect() { } /** - * Test method to verify bidirectional communication. - * Sends a test command to the server, which will request a resource back from the client. + * Test method to verify bidirectional communication. Sends a test command to the server, which will request a + * resource back from the client. * * @param resourceName the resource name to use for the test (e.g., "com/example/Test.java") */ @@ -241,8 +240,8 @@ private void closeStream() { } /** - * Event details for a resource request from the server. - * Wraps the proto RemoteFileSourceMetaRequest and provides a respond() method. + * Event details for a resource request from the server. Wraps the proto RemoteFileSourceMetaRequest and provides a + * respond() method. */ @TsInterface @TsName(namespace = "dh.remotefilesource", name = "ResourceRequest") @@ -351,11 +350,16 @@ private static Uint8Array stringToUtf8(String str) { } private static int sizeOfVarint(int value) { - if (value < 0) return 10; - if (value < 128) return 1; - if (value < 16384) return 2; - if (value < 2097152) return 3; - if (value < 268435456) return 4; + if (value < 0) + return 10; + if (value < 128) + return 1; + if (value < 16384) + return 2; + if (value < 2097152) + return 3; + if (value < 268435456) + return 4; return 5; } From 361019a8ccc379623002bf7a984ed42824c02024 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Thu, 4 Dec 2025 15:46:05 -0600 Subject: [PATCH 12/57] set execution context (#DH-20578) --- .../RemoteFileSourceServicePlugin.java | 90 ++++++++++++++++++- .../proto/remotefilesource.proto | 26 +++++- .../JsRemoteFileSourceService.java | 55 +++++++++++- .../RemoteFileSourceClientRequest.java | 10 +++ .../RemoteFileSourceServerRequest.java | 10 +++ .../SetExecutionContextRequest.java | 42 +++++++++ .../SetExecutionContextResponse.java | 37 ++++++++ 7 files changed, 266 insertions(+), 4 deletions(-) create mode 100644 web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java create mode 100644 web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextResponse.java diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java index ca971c51d86..2110f6c71c6 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java @@ -14,6 +14,7 @@ import io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaRequest; import io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaResponse; import io.deephaven.proto.backplane.grpc.RemoteFileSourceServerRequest; +import io.deephaven.proto.backplane.grpc.SetExecutionContextResponse; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; @@ -27,6 +28,18 @@ public class RemoteFileSourceServicePlugin extends ObjectTypeBase { private static final Logger log = LoggerFactory.getLogger(RemoteFileSourceServicePlugin.class); + /** + * The execution context ID that identifies the currently active RemoteFileSourceMessageStream. + * This corresponds to the clientSessionId of the most recent script execution. + */ + private static volatile String executionContextId; + + /** + * Top-level package names that should be resolved from the remote source + * (e.g., ["com.example", "org.mycompany"]). + */ + private static volatile java.util.List topLevelPackages = new java.util.ArrayList<>(); + private volatile RemoteFileSourceMessageStream messageStream; private final String clientSessionId; @@ -42,6 +55,57 @@ public RemoteFileSourceServicePlugin(String clientSessionId) { } } + /** + * Gets the current execution context ID. + * + * @return the execution context ID, or null if not set + */ + public static String getExecutionContextId() { + return executionContextId; + } + + /** + * Gets the top-level package names that should be resolved from the remote source. + * + * @return the list of top-level package names + */ + public static java.util.List getTopLevelPackages() { + return new java.util.ArrayList<>(topLevelPackages); + } + + /** + * Sets the execution context ID to identify the currently active RemoteFileSourceMessageStream. + * This should be called when a script execution begins to indicate which client session should + * provide source files. + * + * @param contextId the execution context ID (typically matches a clientSessionId) + * @param packages list of top-level package names to resolve from remote source + */ + public static void setExecutionContextId(String contextId, java.util.List packages) { + executionContextId = contextId; + topLevelPackages = packages != null ? new java.util.ArrayList<>(packages) : new java.util.ArrayList<>(); + log.info().append("Execution context ID set to: ").append(contextId) + .append(" with packages: ").append(String.join(", ", topLevelPackages)).endl(); + } + + /** + * Sets the execution context ID without updating packages (backwards compatibility). + * + * @param contextId the execution context ID (typically matches a clientSessionId) + */ + public static void setExecutionContextId(String contextId) { + setExecutionContextId(contextId, java.util.Collections.emptyList()); + } + + /** + * Clears the execution context ID and top-level packages. + */ + public static void clearExecutionContextId() { + executionContextId = null; + topLevelPackages = new java.util.ArrayList<>(); + log.info().append("Execution context ID cleared").endl(); + } + @Override public String name() { return "RemoteFileSourceService"; @@ -80,7 +144,7 @@ public void testRequestResource(String resourceName) { /** * A message stream for the RemoteFileSourceService. */ - private static class RemoteFileSourceMessageStream implements MessageStream { + public static class RemoteFileSourceMessageStream implements MessageStream { private final MessageStream connection; private final Map> pendingRequests = new ConcurrentHashMap<>(); private final String connectionId; @@ -142,6 +206,30 @@ public void onData(ByteBuffer payload, Object... references) throws ObjectCommun log.info().append("Client initiated test for resource: ").append(resourceName).endl(); testRequestResource(resourceName); } + } else if (message.hasSetExecutionContext()) { + // Client is setting the execution context ID + String contextId = message.getSetExecutionContext().getExecutionContextId(); + java.util.List packages = message.getSetExecutionContext().getTopLevelPackagesList(); + setExecutionContextId(contextId, packages); + log.info().append("Client set execution context ID to: ").append(contextId) + .append(" with ").append(packages.size()).append(" top-level packages").endl(); + + // Send acknowledgment back to client + SetExecutionContextResponse response = SetExecutionContextResponse.newBuilder() + .setExecutionContextId(contextId) + .setSuccess(true) + .build(); + + RemoteFileSourceServerRequest serverRequest = RemoteFileSourceServerRequest.newBuilder() + .setRequestId(requestId) + .setSetExecutionContextResponse(response) + .build(); + + try { + connection.onData(ByteBuffer.wrap(serverRequest.toByteArray())); + } catch (ObjectCommunicationException e) { + log.error().append("Failed to send execution context acknowledgment: ").append(e).endl(); + } } else { log.warn().append("Received unknown message type from client").endl(); } diff --git a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto index cdfb3e70092..70f62e07396 100644 --- a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto +++ b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto @@ -19,6 +19,9 @@ message RemoteFileSourceServerRequest { oneof request { // Request source data/resource from the client RemoteFileSourceMetaRequest meta_request = 2; + + // Acknowledgment that execution context was set + SetExecutionContextResponse set_execution_context_response = 3; } } @@ -31,8 +34,11 @@ message RemoteFileSourceClientRequest { // Response to a resource request RemoteFileSourceMetaResponse meta_response = 2; + // Set the execution context ID for script execution + SetExecutionContextRequest set_execution_context = 3; + // Test command (e.g., "TEST:com/example/Test.java") - client triggers server to request a resource back - string test_command = 3; + string test_command = 4; } } @@ -54,6 +60,24 @@ message RemoteFileSourceMetaResponse { string error = 3; } +// Request to set the execution context ID for script execution +message SetExecutionContextRequest { + // The execution context ID to set (typically matches the clientSessionId) + string execution_context_id = 1; + + // Top-level package names that should be resolved from the remote source + // (e.g., ["com.example", "org.mycompany"]) + repeated string top_level_packages = 2; +} + +// Response acknowledging execution context was set +message SetExecutionContextResponse { + // The execution context ID that was set + string execution_context_id = 1; + + // Whether the operation was successful + bool success = 2; +} // Fetch the remote file source plugin into the specified ticket (Flight command, not MessageStream) message RemoteFileSourcePluginFetchRequest { diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index 44c48ebce58..bd9fc1a824a 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -19,6 +19,8 @@ import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceMetaResponse; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourcePluginFetchRequest; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceServerRequest; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.SetExecutionContextRequest; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.SetExecutionContextResponse; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.Ticket; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.TypedTicket; import io.deephaven.web.client.api.Callbacks; @@ -27,9 +29,12 @@ import io.deephaven.web.client.api.event.HasEventHandling; import jsinterop.annotations.JsIgnore; import jsinterop.annotations.JsMethod; +import jsinterop.annotations.JsOptional; import jsinterop.annotations.JsProperty; import jsinterop.base.Js; +import java.util.HashMap; +import java.util.Map; import java.util.function.Supplier; /** @@ -53,6 +58,10 @@ public class JsRemoteFileSourceService extends HasEventHandling { private boolean hasFetched; + // Track pending setExecutionContext requests + private final Map> pendingSetExecutionContextRequests = new HashMap<>(); + private int requestIdCounter = 0; + @JsIgnore private JsRemoteFileSourceService(WorkerConnection connection, TypedTicket typedTicket) { this.typedTicket = typedTicket; @@ -161,6 +170,15 @@ private Promise connect() { // Fire request event (include request_id from wrapper) DomGlobal.setTimeout(ignore -> fireEvent(EVENT_REQUEST, new ResourceRequestEvent(message.getRequestId(), request)), 0); + } else if (message.hasSetExecutionContextResponse()) { + // Server acknowledged execution context + String requestId = message.getRequestId(); + Promise.PromiseExecutorCallbackFn.ResolveCallbackFn resolveCallback = + pendingSetExecutionContextRequests.remove(requestId); + if (resolveCallback != null) { + SetExecutionContextResponse response = message.getSetExecutionContextResponse(); + resolveCallback.onInvoke(response.getSuccess()); + } } else { // Unknown message type DomGlobal.setTimeout(ignore -> fireEvent(EVENT_MESSAGE, res.getData()), 0); @@ -192,8 +210,8 @@ private Promise connect() { } /** - * Test method to verify bidirectional communication. Sends a test command to the server, which will request a - * resource back from the client. + * Test method to verify bidirectional communication. + * Sends a test command to the server, which will request a resource back from the client. * * @param resourceName the resource name to use for the test (e.g., "com/example/Test.java") */ @@ -205,6 +223,39 @@ public void testBidirectionalCommunication(String resourceName) { sendClientRequest(clientRequest); } + /** + * Sets the execution context ID on the server to identify which client session should + * provide source files for script execution. + * + * @param executionContextId the execution context ID (typically matches the client session ID) + * @param topLevelPackages array of top-level package names to resolve from remote source (e.g., ["com.example", "org.mycompany"]) + * @return a promise that resolves to true if the server successfully set the execution context, false otherwise + */ + @JsMethod + public Promise setExecutionContext(String executionContextId, @JsOptional String[] topLevelPackages) { + return new Promise<>((resolve, reject) -> { + // Generate a unique request ID + String requestId = "setExecutionContext-" + (requestIdCounter++); + + // Store the resolve callback to call when we get the acknowledgment + pendingSetExecutionContextRequests.put(requestId, resolve); + + SetExecutionContextRequest setContextRequest = new SetExecutionContextRequest(); + setContextRequest.setExecutionContextId(executionContextId); + + if (topLevelPackages != null && topLevelPackages.length > 0) { + for (String pkg : topLevelPackages) { + setContextRequest.addTopLevelPackages(pkg); + } + } + + RemoteFileSourceClientRequest clientRequest = new RemoteFileSourceClientRequest(); + clientRequest.setRequestId(requestId); + clientRequest.setSetExecutionContext(setContextRequest); + sendClientRequest(clientRequest); + }); + } + /** * Helper method to send a RemoteFileSourceClientRequest to the server. * diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java index 28d6736b152..c6426ab86d5 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java @@ -26,16 +26,22 @@ public static native void serializeBinaryToWriter( public native void clearTestCommand(); + public native void clearSetExecutionContext(); + public native String getRequestId(); public native RemoteFileSourceMetaResponse getMetaResponse(); public native String getTestCommand(); + public native SetExecutionContextRequest getSetExecutionContext(); + public native boolean hasMetaResponse(); public native boolean hasTestCommand(); + public native boolean hasSetExecutionContext(); + public native Uint8Array serializeBinary(); public native void setRequestId(String value); @@ -46,5 +52,9 @@ public static native void serializeBinaryToWriter( public native void setTestCommand(String value); + + public native void setSetExecutionContext(); + + public native void setSetExecutionContext(SetExecutionContextRequest value); } diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java index 2df13fcdd17..4ce84895ce4 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java @@ -24,12 +24,18 @@ public static native void serializeBinaryToWriter( public native void clearMetaRequest(); + public native void clearSetExecutionContextResponse(); + public native String getRequestId(); public native RemoteFileSourceMetaRequest getMetaRequest(); + public native SetExecutionContextResponse getSetExecutionContextResponse(); + public native boolean hasMetaRequest(); + public native boolean hasSetExecutionContextResponse(); + public native Uint8Array serializeBinary(); public native void setRequestId(String value); @@ -37,5 +43,9 @@ public static native void serializeBinaryToWriter( public native void setMetaRequest(); public native void setMetaRequest(RemoteFileSourceMetaRequest value); + + public native void setSetExecutionContextResponse(); + + public native void setSetExecutionContextResponse(SetExecutionContextResponse value); } diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java new file mode 100644 index 00000000000..200472e155f --- /dev/null +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java @@ -0,0 +1,42 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; + +import elemental2.core.JsArray; +import elemental2.core.Uint8Array; +import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsType; + +@JsType( + isNative = true, + name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.SetExecutionContextRequest", + namespace = JsPackage.GLOBAL) +public class SetExecutionContextRequest { + public static native SetExecutionContextRequest deserializeBinary(Uint8Array bytes); + + public static native SetExecutionContextRequest deserializeBinaryFromReader( + SetExecutionContextRequest message, Object reader); + + public static native void serializeBinaryToWriter( + SetExecutionContextRequest message, Object writer); + + public native void clearExecutionContextId(); + + public native void clearTopLevelPackagesList(); + + public native String getExecutionContextId(); + + public native JsArray getTopLevelPackagesList(); + + public native Uint8Array serializeBinary(); + + public native void setExecutionContextId(String value); + + public native void setTopLevelPackagesList(JsArray value); + + public native void addTopLevelPackages(String value); + + public native void addTopLevelPackages(String value, double index); +} + diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextResponse.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextResponse.java new file mode 100644 index 00000000000..8b43eabcbfb --- /dev/null +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextResponse.java @@ -0,0 +1,37 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; + +import elemental2.core.Uint8Array; +import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsType; + +@JsType( + isNative = true, + name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.SetExecutionContextResponse", + namespace = JsPackage.GLOBAL) +public class SetExecutionContextResponse { + public static native SetExecutionContextResponse deserializeBinary(Uint8Array bytes); + + public static native SetExecutionContextResponse deserializeBinaryFromReader( + SetExecutionContextResponse message, Object reader); + + public static native void serializeBinaryToWriter( + SetExecutionContextResponse message, Object writer); + + public native void clearExecutionContextId(); + + public native void clearSuccess(); + + public native String getExecutionContextId(); + + public native boolean getSuccess(); + + public native Uint8Array serializeBinary(); + + public native void setExecutionContextId(String value); + + public native void setSuccess(boolean value); +} + From 2a1525aa96ea84fa7ae0057a8fbe1fa32036bede Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Fri, 5 Dec 2025 10:34:00 -0600 Subject: [PATCH 13/57] Simplified execution context (#DH-20578) --- .../RemoteFileSourceCommandResolver.java | 10 +- .../RemoteFileSourceServicePlugin.java | 153 +++++++++--------- .../proto/remotefilesource.proto | 17 +- .../deephaven/web/client/api/CoreClient.java | 4 +- .../JsRemoteFileSourceService.java | 14 +- .../SetExecutionContextRequest.java | 5 - .../SetExecutionContextResponse.java | 5 - 7 files changed, 89 insertions(+), 119 deletions(-) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java index 12276dc5380..5e23ec3fb5b 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java @@ -58,24 +58,20 @@ private static Any parseOrNull(final ByteString data) { } public SessionState.ExportObject fetchPlugin(@Nullable final SessionState session, - final Flight.FlightDescriptor descriptor, - final RemoteFileSourcePluginFetchRequest request) { + final Flight.FlightDescriptor descriptor, + final RemoteFileSourcePluginFetchRequest request) { final Ticket resultTicket = request.getResultId(); final boolean hasResultId = !resultTicket.getTicket().isEmpty(); if (!hasResultId) { throw new StatusRuntimeException(Status.INVALID_ARGUMENT); } - // Extract optional client session ID from the request (empty string means not provided) - final String clientSessionId = request.getClientSessionId(); - final SessionState.ExportBuilder pluginExportBuilder = session.newExport(resultTicket, "RemoteFileSourcePluginFetchRequest.resultTicket"); pluginExportBuilder.require(); final SessionState.ExportObject pluginExport = - pluginExportBuilder.submit(() -> new RemoteFileSourceServicePlugin( - clientSessionId.isEmpty() ? null : clientSessionId)); + pluginExportBuilder.submit(RemoteFileSourceServicePlugin::new); final Flight.FlightInfo flightInfo = Flight.FlightInfo.newBuilder() .setFlightDescriptor(descriptor) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java index 2110f6c71c6..2c3133bcabf 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java @@ -29,81 +29,48 @@ public class RemoteFileSourceServicePlugin extends ObjectTypeBase { private static final Logger log = LoggerFactory.getLogger(RemoteFileSourceServicePlugin.class); /** - * The execution context ID that identifies the currently active RemoteFileSourceMessageStream. - * This corresponds to the clientSessionId of the most recent script execution. + * The current execution context containing the active message stream and configuration. + * Null when no execution context is active. */ - private static volatile String executionContextId; - - /** - * Top-level package names that should be resolved from the remote source - * (e.g., ["com.example", "org.mycompany"]). - */ - private static volatile java.util.List topLevelPackages = new java.util.ArrayList<>(); + private static volatile RemoteFileSourceExecutionContext executionContext; private volatile RemoteFileSourceMessageStream messageStream; - private final String clientSessionId; public RemoteFileSourceServicePlugin() { - this(null); - } - - public RemoteFileSourceServicePlugin(String clientSessionId) { - this.clientSessionId = clientSessionId; - if (clientSessionId != null) { - log.info().append("RemoteFileSourceServicePlugin created with clientSessionId: ").append(clientSessionId) - .endl(); - } } /** - * Gets the current execution context ID. + * Sets the execution context with the active message stream and top-level packages. + * This should be called when a script execution begins. * - * @return the execution context ID, or null if not set - */ - public static String getExecutionContextId() { - return executionContextId; - } - - /** - * Gets the top-level package names that should be resolved from the remote source. - * - * @return the list of top-level package names - */ - public static java.util.List getTopLevelPackages() { - return new java.util.ArrayList<>(topLevelPackages); - } - - /** - * Sets the execution context ID to identify the currently active RemoteFileSourceMessageStream. - * This should be called when a script execution begins to indicate which client session should - * provide source files. - * - * @param contextId the execution context ID (typically matches a clientSessionId) + * @param messageStream the message stream to set as active (must not be null) * @param packages list of top-level package names to resolve from remote source + * @throws IllegalArgumentException if messageStream is null (use clearExecutionContext() instead) */ - public static void setExecutionContextId(String contextId, java.util.List packages) { - executionContextId = contextId; - topLevelPackages = packages != null ? new java.util.ArrayList<>(packages) : new java.util.ArrayList<>(); - log.info().append("Execution context ID set to: ").append(contextId) - .append(" with packages: ").append(String.join(", ", topLevelPackages)).endl(); + public static void setExecutionContext(RemoteFileSourceMessageStream messageStream, java.util.List packages) { + if (messageStream == null) { + throw new IllegalArgumentException("messageStream must not be null. Use clearExecutionContext() to clear the context."); + } + executionContext = new RemoteFileSourceExecutionContext(messageStream, packages); + log.info().append("Set execution context with ") + .append(packages != null ? packages.size() : 0).append(" top-level packages").endl(); } /** - * Sets the execution context ID without updating packages (backwards compatibility). - * - * @param contextId the execution context ID (typically matches a clientSessionId) + * Clears the execution context. */ - public static void setExecutionContextId(String contextId) { - setExecutionContextId(contextId, java.util.Collections.emptyList()); + public static void clearExecutionContext() { + executionContext = null; + log.info().append("Cleared execution context").endl(); } /** - * Clears the execution context ID and top-level packages. + * Gets the current execution context. + * + * @return the execution context */ - public static void clearExecutionContextId() { - executionContextId = null; - topLevelPackages = new java.util.ArrayList<>(); - log.info().append("Execution context ID cleared").endl(); + public static RemoteFileSourceExecutionContext getExecutionContext() { + return executionContext; } @Override @@ -120,7 +87,7 @@ public boolean isType(Object object) { public MessageStream compatibleClientConnection(Object object, MessageStream connection) throws ObjectCommunicationException { connection.onData(ByteBuffer.allocate(0)); - messageStream = new RemoteFileSourceMessageStream(connection, clientSessionId); + messageStream = new RemoteFileSourceMessageStream(connection); return messageStream; } @@ -147,22 +114,9 @@ public void testRequestResource(String resourceName) { public static class RemoteFileSourceMessageStream implements MessageStream { private final MessageStream connection; private final Map> pendingRequests = new ConcurrentHashMap<>(); - private final String connectionId; - public RemoteFileSourceMessageStream(final MessageStream connection, final String clientSessionId) { + public RemoteFileSourceMessageStream(final MessageStream connection) { this.connection = connection; - this.connectionId = clientSessionId; // Initialize with the ID from the fetch request - if (clientSessionId != null) { - log.info().append("RemoteFileSourceMessageStream initialized with clientSessionId: ") - .append(clientSessionId).endl(); - } - } - - /** - * @return the connection ID set by the client, or null if not set - */ - public String getConnectionId() { - return connectionId; } @Override @@ -207,16 +161,14 @@ public void onData(ByteBuffer payload, Object... references) throws ObjectCommun testRequestResource(resourceName); } } else if (message.hasSetExecutionContext()) { - // Client is setting the execution context ID - String contextId = message.getSetExecutionContext().getExecutionContextId(); + // Client is requesting this message stream to become active java.util.List packages = message.getSetExecutionContext().getTopLevelPackagesList(); - setExecutionContextId(contextId, packages); - log.info().append("Client set execution context ID to: ").append(contextId) - .append(" with ").append(packages.size()).append(" top-level packages").endl(); + setExecutionContext(this, packages); + log.info().append("Client set execution context for this message stream with ") + .append(packages.size()).append(" top-level packages").endl(); // Send acknowledgment back to client SetExecutionContextResponse response = SetExecutionContextResponse.newBuilder() - .setExecutionContextId(contextId) .setSuccess(true) .build(); @@ -241,6 +193,12 @@ public void onData(ByteBuffer payload, Object... references) throws ObjectCommun @Override public void onClose() { + // Clear execution context if this was the active stream + RemoteFileSourceExecutionContext context = executionContext; + if (context != null && context.getActiveMessageStream() == this) { + clearExecutionContext(); + } + // Cancel all pending requests pendingRequests.values().forEach(future -> future.cancel(true)); pendingRequests.clear(); @@ -309,4 +267,45 @@ public void testRequestResource(String resourceName) { }); } } + + /** + * Encapsulates the execution context for remote file source operations. + * This includes the currently active message stream and the top-level packages + * that should be resolved from the remote source. + * This class is immutable - a new instance is created each time the context changes. + */ + public static class RemoteFileSourceExecutionContext { + private final RemoteFileSourceMessageStream activeMessageStream; + private final java.util.List topLevelPackages; + + /** + * Creates a new execution context. + * + * @param activeMessageStream the active message stream + * @param topLevelPackages list of top-level package names to resolve from remote source + */ + public RemoteFileSourceExecutionContext(RemoteFileSourceMessageStream activeMessageStream, + java.util.List topLevelPackages) { + this.activeMessageStream = activeMessageStream; + this.topLevelPackages = topLevelPackages != null ? topLevelPackages : java.util.Collections.emptyList(); + } + + /** + * Gets the currently active message stream. + * + * @return the active message stream + */ + public RemoteFileSourceMessageStream getActiveMessageStream() { + return activeMessageStream; + } + + /** + * Gets the top-level package names that should be resolved from the remote source. + * + * @return a copy of the list of top-level package names + */ + public java.util.List getTopLevelPackages() { + return new java.util.ArrayList<>(topLevelPackages); + } + } } diff --git a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto index 70f62e07396..5fef67186f4 100644 --- a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto +++ b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto @@ -60,29 +60,20 @@ message RemoteFileSourceMetaResponse { string error = 3; } -// Request to set the execution context ID for script execution +// Request to set the execution context for script execution message SetExecutionContextRequest { - // The execution context ID to set (typically matches the clientSessionId) - string execution_context_id = 1; - // Top-level package names that should be resolved from the remote source - // (e.g., ["com.example", "org.mycompany"]) - repeated string top_level_packages = 2; + // (e.g., ["com", "org"]) + repeated string top_level_packages = 1; } // Response acknowledging execution context was set message SetExecutionContextResponse { - // The execution context ID that was set - string execution_context_id = 1; - // Whether the operation was successful - bool success = 2; + bool success = 1; } // Fetch the remote file source plugin into the specified ticket (Flight command, not MessageStream) message RemoteFileSourcePluginFetchRequest { io.deephaven.proto.backplane.grpc.Ticket result_id = 1; - - // Optional client session ID to identify this client connection - string client_session_id = 2; } \ No newline at end of file diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java b/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java index 27473a00f5f..ecdeee17289 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java @@ -155,8 +155,8 @@ public JsStorageService getStorageService() { return new JsStorageService(ideConnection.connection.get()); } - public Promise getRemoteFileSourceService(@JsOptional String clientSessionId) { - return JsRemoteFileSourceService.fetchPlugin(ideConnection.connection.get(), clientSessionId); + public Promise getRemoteFileSourceService() { + return JsRemoteFileSourceService.fetchPlugin(ideConnection.connection.get()); } public Promise getAsIdeConnection() { diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index bd9fc1a824a..c9be6bc299e 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -80,20 +80,16 @@ private JsRemoteFileSourceService(WorkerConnection connection, TypedTicket typed * Fetches a RemoteFileSource plugin instance from the server and establishes a message stream connection. * * @param connection the worker connection to use for communication - * @param clientSessionId optional unique identifier for this client session * @return a promise that resolves to a RemoteFileSourceService instance with an active message stream */ @JsMethod - public static Promise fetchPlugin(WorkerConnection connection, String clientSessionId) { + public static Promise fetchPlugin(WorkerConnection connection) { // Create a new export ticket for the result Ticket resultTicket = connection.getTickets().newExportTicket(); // Create the fetch request RemoteFileSourcePluginFetchRequest fetchRequest = new RemoteFileSourcePluginFetchRequest(); fetchRequest.setResultId(resultTicket); - if (clientSessionId != null && !clientSessionId.isEmpty()) { - fetchRequest.setClientSessionId(clientSessionId); - } // Serialize the request to bytes Uint8Array innerRequestBytes = fetchRequest.serializeBinary(); @@ -224,15 +220,14 @@ public void testBidirectionalCommunication(String resourceName) { } /** - * Sets the execution context ID on the server to identify which client session should - * provide source files for script execution. + * Sets the execution context on the server to identify this message stream as active + * for script execution. * - * @param executionContextId the execution context ID (typically matches the client session ID) * @param topLevelPackages array of top-level package names to resolve from remote source (e.g., ["com.example", "org.mycompany"]) * @return a promise that resolves to true if the server successfully set the execution context, false otherwise */ @JsMethod - public Promise setExecutionContext(String executionContextId, @JsOptional String[] topLevelPackages) { + public Promise setExecutionContext(@JsOptional String[] topLevelPackages) { return new Promise<>((resolve, reject) -> { // Generate a unique request ID String requestId = "setExecutionContext-" + (requestIdCounter++); @@ -241,7 +236,6 @@ public Promise setExecutionContext(String executionContextId, @JsOption pendingSetExecutionContextRequests.put(requestId, resolve); SetExecutionContextRequest setContextRequest = new SetExecutionContextRequest(); - setContextRequest.setExecutionContextId(executionContextId); if (topLevelPackages != null && topLevelPackages.length > 0) { for (String pkg : topLevelPackages) { diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java index 200472e155f..3748185297b 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java @@ -21,17 +21,12 @@ public static native SetExecutionContextRequest deserializeBinaryFromReader( public static native void serializeBinaryToWriter( SetExecutionContextRequest message, Object writer); - public native void clearExecutionContextId(); - public native void clearTopLevelPackagesList(); - public native String getExecutionContextId(); - public native JsArray getTopLevelPackagesList(); public native Uint8Array serializeBinary(); - public native void setExecutionContextId(String value); public native void setTopLevelPackagesList(JsArray value); diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextResponse.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextResponse.java index 8b43eabcbfb..8c27b48e69a 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextResponse.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextResponse.java @@ -20,17 +20,12 @@ public static native SetExecutionContextResponse deserializeBinaryFromReader( public static native void serializeBinaryToWriter( SetExecutionContextResponse message, Object writer); - public native void clearExecutionContextId(); - public native void clearSuccess(); - public native String getExecutionContextId(); - public native boolean getSuccess(); public native Uint8Array serializeBinary(); - public native void setExecutionContextId(String value); public native void setSuccess(boolean value); } From 36143fa702a8cd9c05877f4228afd8ecfd8f23f3 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Mon, 8 Dec 2025 17:31:06 -0600 Subject: [PATCH 14/57] Basic file sourcing is working (#DH-20578) --- .../util/RemoteFileSourceClassLoader.java | 119 ++++++++++++++++++ .../engine/util/RemoteFileSourceProvider.java | 37 ++++++ .../engine/util/GroovyDeephavenSession.java | 2 +- .../RemoteFileSourceServicePlugin.java | 82 +++++++++++- 4 files changed, 236 insertions(+), 4 deletions(-) create mode 100644 engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java create mode 100644 engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java new file mode 100644 index 00000000000..b7d65b2bc07 --- /dev/null +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java @@ -0,0 +1,119 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.engine.util; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.net.URLConnection; +import java.net.URLStreamHandler; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; + +/** + * A custom ClassLoader that fetches source files from remote clients via registered RemoteFileSourceProvider instances. + * This is designed to support Groovy script imports where the source files are provided by remote clients. + * + *

When a resource is requested (e.g., for a Groovy import), this class loader: + *

    + *
  1. Checks registered providers to see if they can source the resource
    2. + *
  2. Returns a custom URL with protocol "remotefile://" if a provider can handle it
    3. + *
  3. When that URL is opened, fetches the resource bytes from the provider
    4. + *
+ */ +public class RemoteFileSourceClassLoader extends ClassLoader { + private static final boolean DEBUG = Boolean.getBoolean("RemoteFileSourceClassLoader.debug"); + private static volatile RemoteFileSourceClassLoader instance; + private final CopyOnWriteArrayList providers = new CopyOnWriteArrayList<>(); + + public RemoteFileSourceClassLoader(ClassLoader parent) { + super(parent); + instance = this; + } + + public static RemoteFileSourceClassLoader getInstance() { + return instance; + } + + public void registerProvider(RemoteFileSourceProvider provider) { + providers.add(provider); + } + + @Override + protected URL findResource(String name) { + for (RemoteFileSourceProvider provider : providers) { + if (!provider.isActive()) { + continue; + } + try { + Boolean canSource = provider.canSourceResource(name) + .orTimeout(1, TimeUnit.SECONDS) + .get(); + if (Boolean.TRUE.equals(canSource)) { + return new URL(null, "remotefile://" + name, new RemoteFileURLStreamHandler(provider, name)); + } + } catch (Exception e) { + // Continue to next provider + } + } + return super.findResource(name); + } + + /** + * URLStreamHandler that delegates to a RemoteFileSourceProvider to fetch resource bytes. + */ + private static class RemoteFileURLStreamHandler extends URLStreamHandler { + private final RemoteFileSourceProvider provider; + private final String resourceName; + + RemoteFileURLStreamHandler(RemoteFileSourceProvider provider, String resourceName) { + this.provider = provider; + this.resourceName = resourceName; + } + + @Override + protected URLConnection openConnection(URL u) { + return new RemoteFileURLConnection(u, provider, resourceName); + } + } + + /** + * URLConnection that fetches resource bytes from a RemoteFileSourceProvider. + */ + private static class RemoteFileURLConnection extends URLConnection { + private final RemoteFileSourceProvider provider; + private final String resourceName; + private byte[] content; + + RemoteFileURLConnection(URL url, RemoteFileSourceProvider provider, String resourceName) { + super(url); + this.provider = provider; + this.resourceName = resourceName; + } + + @Override + public void connect() throws IOException { + if (!connected) { + try { + content = provider.requestResource(resourceName) + .orTimeout(5, TimeUnit.SECONDS) + .get(); + connected = true; + } catch (Exception e) { + throw new IOException("Failed to fetch remote resource: " + resourceName, e); + } + } + } + + @Override + public InputStream getInputStream() throws IOException { + connect(); + if (content == null || content.length == 0) { + throw new IOException("No content for resource: " + resourceName); + } + return new ByteArrayInputStream(content); + } + } +} diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java new file mode 100644 index 00000000000..4c5408d7833 --- /dev/null +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java @@ -0,0 +1,37 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.engine.util; + +import java.util.concurrent.CompletableFuture; + +/** + * Interface for providing remote resources to the ClassLoader. + * Plugins can implement this interface and register with RemoteFileSourceClassLoader + * to provide resources from remote sources. + */ +public interface RemoteFileSourceProvider { + /** + * Check if this provider can source the given resource. + * + * @param resourceName the name of the resource to check (e.g., "com/example/MyClass.groovy") + * @return a CompletableFuture that resolves to true if this provider can handle the resource, false otherwise + */ + CompletableFuture canSourceResource(String resourceName); + + /** + * Request a resource from the remote source. + * + * @param resourceName the name of the resource to fetch (e.g., "com/example/MyClass.groovy") + * @return a CompletableFuture containing the resource bytes, or null if not found + */ + CompletableFuture requestResource(String resourceName); + + /** + * Check if this provider is currently active and should be used for resource requests. + * + * @return true if this provider is active, false otherwise + */ + boolean isActive(); +} + diff --git a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java index 569de649341..91dcdf88187 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java @@ -96,7 +96,7 @@ public class GroovyDeephavenSession extends AbstractScriptSession mapping = new ConcurrentHashMap<>(); @Override diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java index 2c3133bcabf..ff733394c5e 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java @@ -25,7 +25,7 @@ import java.util.concurrent.TimeUnit; @AutoService(ObjectType.class) -public class RemoteFileSourceServicePlugin extends ObjectTypeBase { +public class RemoteFileSourceServicePlugin extends ObjectTypeBase implements io.deephaven.engine.util.RemoteFileSourceProvider { private static final Logger log = LoggerFactory.getLogger(RemoteFileSourceServicePlugin.class); /** @@ -37,11 +37,65 @@ public class RemoteFileSourceServicePlugin extends ObjectTypeBase { private volatile RemoteFileSourceMessageStream messageStream; public RemoteFileSourceServicePlugin() { + log.info().append("🎯 RemoteFileSourceServicePlugin constructor called").endl(); + // Register eagerly with the class loader + registerWithClassLoader(); + } + + // RemoteFileSourceProvider interface implementation - delegates to active message stream + + @Override + public CompletableFuture canSourceResource(String resourceName) { + // Only handle .groovy source files, not compiled .class files + if (!resourceName.endsWith(".groovy")) { + return CompletableFuture.completedFuture(false); + } + + RemoteFileSourceExecutionContext context = executionContext; + if (context == null) { + return CompletableFuture.completedFuture(false); + } + + java.util.List topLevelPackages = context.getTopLevelPackages(); + if (topLevelPackages.isEmpty()) { + return CompletableFuture.completedFuture(false); + } + + String resourcePath = resourceName.replace('\\', '/'); + + for (String topLevelPackage : topLevelPackages) { + String packagePath = topLevelPackage.replace('.', '/'); + if (resourcePath.startsWith(packagePath + "/") || resourcePath.startsWith(packagePath)) { + log.info().append("✅ Can source: ").append(resourceName).endl(); + return CompletableFuture.completedFuture(true); + } + } + + return CompletableFuture.completedFuture(false); + } + + @Override + public CompletableFuture requestResource(String resourceName) { + log.info().append("📥 Requesting resource: ").append(resourceName).endl(); + + RemoteFileSourceExecutionContext context = executionContext; + if (context == null) { + log.warn().append("No execution context when requesting resource").endl(); + return CompletableFuture.completedFuture(null); + } + + return context.getActiveMessageStream().requestResource(resourceName); + } + + @Override + public boolean isActive() { + return executionContext != null; } /** * Sets the execution context with the active message stream and top-level packages. * This should be called when a script execution begins. + * The plugin (which is registered with the ClassLoader) will route requests to this message stream. * * @param messageStream the message stream to set as active (must not be null) * @param packages list of top-level package names to resolve from remote source @@ -51,6 +105,8 @@ public static void setExecutionContext(RemoteFileSourceMessageStream messageStre if (messageStream == null) { throw new IllegalArgumentException("messageStream must not be null. Use clearExecutionContext() to clear the context."); } + + // Set new context - the plugin will automatically route to this message stream executionContext = new RemoteFileSourceExecutionContext(messageStream, packages); log.info().append("Set execution context with ") .append(packages != null ? packages.size() : 0).append(" top-level packages").endl(); @@ -60,10 +116,29 @@ public static void setExecutionContext(RemoteFileSourceMessageStream messageStre * Clears the execution context. */ public static void clearExecutionContext() { - executionContext = null; - log.info().append("Cleared execution context").endl(); + if (executionContext != null) { + executionContext = null; + log.info().append("Cleared execution context").endl(); + } + } + + /** + * Register this plugin instance with the RemoteFileSourceClassLoader instance. + * Called once during plugin construction. + */ + private void registerWithClassLoader() { + io.deephaven.engine.util.RemoteFileSourceClassLoader classLoader = + io.deephaven.engine.util.RemoteFileSourceClassLoader.getInstance(); + + if (classLoader != null) { + classLoader.registerProvider(this); + log.info().append("✅ Registered RemoteFileSourceServicePlugin with RemoteFileSourceClassLoader").endl(); + } else { + log.warn().append("⚠️ RemoteFileSourceClassLoader instance not found - plugin not registered").endl(); + } } + /** * Gets the current execution context. * @@ -241,6 +316,7 @@ public CompletableFuture requestResource(String resourceName) { return future; } + /** * Test method to request a resource and log the result. This can be called from the server console to test the * bidirectional communication. From 82460361eca795cbe1d98f46c22ed4c4451b6b42 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 10 Dec 2025 09:21:38 -0600 Subject: [PATCH 15/57] Comments and cleanup (#DH-20578) --- .../RemoteFileSourceCommandResolver.java | 48 ++++++++++++++++--- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java index 5e23ec3fb5b..f41da7366e1 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java @@ -33,22 +33,34 @@ public class RemoteFileSourceCommandResolver implements CommandResolver, WantsTi private static final String FETCH_PLUGIN_TYPE_URL = "type.googleapis.com/" + RemoteFileSourcePluginFetchRequest.getDescriptor().getFullName(); - + /** + * Parses a RemoteFileSourcePluginFetchRequest from the given Any command. + * + * @param command the Any command containing the fetch request + * @return the parsed RemoteFileSourcePluginFetchRequest + * @throws IllegalArgumentException if the command type URL doesn't match the expected fetch plugin type + * @throws UncheckedDeephavenException if the command cannot be parsed as a RemoteFileSourcePluginFetchRequest + */ private static RemoteFileSourcePluginFetchRequest parseFetchRequest(final Any command) { if (!FETCH_PLUGIN_TYPE_URL.equals(command.getTypeUrl())) { throw new IllegalArgumentException("Not a valid remotefilesource command: " + command.getTypeUrl()); } - final ByteString bytes = command.getValue(); - final RemoteFileSourcePluginFetchRequest request; try { - request = RemoteFileSourcePluginFetchRequest.parseFrom(bytes); + return RemoteFileSourcePluginFetchRequest.parseFrom(command.getValue()); } catch (InvalidProtocolBufferException e) { throw new UncheckedDeephavenException("Could not parse RemoteFileSourcePluginFetchRequest", e); } - return request; } + /** + * Attempts to parse ByteString data as a protobuf Any message. + * Returns null if parsing fails rather than throwing an exception, allowing callers to handle + * invalid data gracefully. + * + * @param data the ByteString data to parse + * @return the parsed Any message, or null if parsing fails + */ private static Any parseOrNull(final ByteString data) { try { return Any.parseFrom(data); @@ -57,6 +69,17 @@ private static Any parseOrNull(final ByteString data) { } } + /** + * Creates and exports a RemoteFileSourceServicePlugin instance based on the fetch request. + * The plugin is exported to the session using the result ticket specified in the request, + * and flight info is returned containing the endpoint for accessing the plugin. + * + * @param session the session state for the current request + * @param descriptor the flight descriptor containing the command + * @param request the parsed RemoteFileSourcePluginFetchRequest containing the result ticket + * @return a FlightInfo export object containing the plugin endpoint information + * @throws StatusRuntimeException if the request doesn't contain a valid result ID ticket + */ public SessionState.ExportObject fetchPlugin(@Nullable final SessionState session, final Flight.FlightDescriptor descriptor, final RemoteFileSourcePluginFetchRequest request) { @@ -86,6 +109,18 @@ public SessionState.ExportObject fetchPlugin(@Nullable final return SessionState.wrapAsExport(flightInfo); } + /** + * Resolves a flight descriptor to flight info for remote file source commands. + * Handles RemoteFileSourcePluginFetchRequest commands by parsing the descriptor and delegating to the + * appropriate handler method. + * + * @param session the session state for the current request + * @param descriptor the flight descriptor containing the command + * @param logId the log identifier for tracking + * @return a FlightInfo export object for the requested command + * @throws StatusRuntimeException if session is null (UNAUTHENTICATED), the command cannot be parsed, + * or the command type URL is not recognized + */ @Override public SessionState.ExportObject flightInfoFor(@Nullable final SessionState session, final Flight.FlightDescriptor descriptor, @@ -156,7 +191,8 @@ public SessionState.ExportBuilder publish(final SessionState session, public SessionState.ExportObject resolve(@Nullable final SessionState session, final Flight.FlightDescriptor descriptor, final String logId) { - return null; + // use flightInfoFor() instead of resolve() for descriptor handling + throw new UnsupportedOperationException(); } @Override From 1998da334368684c04525d3e238735078c12283c Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 10 Dec 2025 09:26:14 -0600 Subject: [PATCH 16/57] Made method private (#DH-20578) --- .../RemoteFileSourceCommandResolver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java index f41da7366e1..32ff1058b9b 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java @@ -80,7 +80,7 @@ private static Any parseOrNull(final ByteString data) { * @return a FlightInfo export object containing the plugin endpoint information * @throws StatusRuntimeException if the request doesn't contain a valid result ID ticket */ - public SessionState.ExportObject fetchPlugin(@Nullable final SessionState session, + private SessionState.ExportObject fetchPlugin(@Nullable final SessionState session, final Flight.FlightDescriptor descriptor, final RemoteFileSourcePluginFetchRequest request) { final Ticket resultTicket = request.getResultId(); From 74e193e20c8e6b121b7add659f9d60eeee44440a Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 10 Dec 2025 09:28:35 -0600 Subject: [PATCH 17/57] Made method static (#DH-20578) --- .../RemoteFileSourceCommandResolver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java index 32ff1058b9b..4475711928c 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java @@ -80,7 +80,7 @@ private static Any parseOrNull(final ByteString data) { * @return a FlightInfo export object containing the plugin endpoint information * @throws StatusRuntimeException if the request doesn't contain a valid result ID ticket */ - private SessionState.ExportObject fetchPlugin(@Nullable final SessionState session, + private static SessionState.ExportObject fetchPlugin(@Nullable final SessionState session, final Flight.FlightDescriptor descriptor, final RemoteFileSourcePluginFetchRequest request) { final Ticket resultTicket = request.getResultId(); From 59afc0721f91df3ab0cae4a5c9f04a9613baaba0 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 10 Dec 2025 11:56:32 -0600 Subject: [PATCH 18/57] onResourceRequest method (#DH-20578) --- .../engine/util/RemoteFileSourceClassLoader.java | 2 +- .../JsRemoteFileSourceService.java | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java index b7d65b2bc07..4e8d631d2d5 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java @@ -49,7 +49,7 @@ protected URL findResource(String name) { } try { Boolean canSource = provider.canSourceResource(name) - .orTimeout(1, TimeUnit.SECONDS) + .orTimeout(5, TimeUnit.SECONDS) .get(); if (Boolean.TRUE.equals(canSource)) { return new URL(null, "remotefile://" + name, new RemoteFileURLStreamHandler(provider, name)); diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index c9be6bc299e..0e7d9ccd889 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -26,7 +26,9 @@ import io.deephaven.web.client.api.Callbacks; import io.deephaven.web.client.api.WorkerConnection; import io.deephaven.web.client.api.barrage.stream.BiDiStream; +import io.deephaven.web.client.api.event.EventFn; import io.deephaven.web.client.api.event.HasEventHandling; +import io.deephaven.web.shared.fu.RemoverFn; import jsinterop.annotations.JsIgnore; import jsinterop.annotations.JsMethod; import jsinterop.annotations.JsOptional; @@ -268,6 +270,18 @@ private void sendClientRequest(RemoteFileSourceClientRequest clientRequest) { messageStream.send(req); } + /** + * Registers a listener for resource requests from the server. + * The listener will be called when the server requests a resource from the client. + * + * @param callback the callback to invoke when a resource is requested + * @return a cleanup function that can be called to remove the listener + */ + @JsMethod + public RemoverFn onResourceRequest(EventFn callback) { + return this.addEventListener(EVENT_REQUEST, callback); + } + /** * Closes the message stream connection to the server. */ From af3eef0ea124cb25bfdf732b8250f5d5edcbcdcc Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 10 Dec 2025 15:58:56 -0600 Subject: [PATCH 19/57] JS API types (#DH-20578) --- .../JsRemoteFileSourceService.java | 30 +++++-------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index 0e7d9ccd889..0aba6fe8614 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -5,6 +5,7 @@ import com.vertispan.tsdefs.annotations.TsInterface; import com.vertispan.tsdefs.annotations.TsName; +import com.vertispan.tsdefs.annotations.TsTypeRef; import elemental2.core.Uint8Array; import elemental2.dom.DomGlobal; import elemental2.promise.Promise; @@ -26,31 +27,28 @@ import io.deephaven.web.client.api.Callbacks; import io.deephaven.web.client.api.WorkerConnection; import io.deephaven.web.client.api.barrage.stream.BiDiStream; -import io.deephaven.web.client.api.event.EventFn; import io.deephaven.web.client.api.event.HasEventHandling; -import io.deephaven.web.shared.fu.RemoverFn; import jsinterop.annotations.JsIgnore; import jsinterop.annotations.JsMethod; +import jsinterop.annotations.JsNullable; import jsinterop.annotations.JsOptional; import jsinterop.annotations.JsProperty; +import jsinterop.annotations.JsType; import jsinterop.base.Js; import java.util.HashMap; import java.util.Map; import java.util.function.Supplier; + /** * JavaScript client for the RemoteFileSource service. Provides bidirectional communication with the server-side * RemoteFileSourceServicePlugin via a message stream. */ -@TsInterface -@TsName(namespace = "dh.remotefilesource", name = "RemoteFileSourceService") +@JsType(namespace = "dh.remotefilesource", name = "RemoteFileSourceService") public class JsRemoteFileSourceService extends HasEventHandling { - @JsProperty(namespace = "dh.remotefilesource.RemoteFileSourceService") public static final String EVENT_MESSAGE = "message"; - @JsProperty(namespace = "dh.remotefilesource.RemoteFileSourceService") public static final String EVENT_CLOSE = "close"; - @JsProperty(namespace = "dh.remotefilesource.RemoteFileSourceService") public static final String EVENT_REQUEST = "request"; private final TypedTicket typedTicket; @@ -85,7 +83,7 @@ private JsRemoteFileSourceService(WorkerConnection connection, TypedTicket typed * @return a promise that resolves to a RemoteFileSourceService instance with an active message stream */ @JsMethod - public static Promise fetchPlugin(WorkerConnection connection) { + public static Promise fetchPlugin(@TsTypeRef(Object.class) WorkerConnection connection) { // Create a new export ticket for the result Ticket resultTicket = connection.getTickets().newExportTicket(); @@ -270,18 +268,6 @@ private void sendClientRequest(RemoteFileSourceClientRequest clientRequest) { messageStream.send(req); } - /** - * Registers a listener for resource requests from the server. - * The listener will be called when the server requests a resource from the client. - * - * @param callback the callback to invoke when a resource is requested - * @return a cleanup function that can be called to remove the listener - */ - @JsMethod - public RemoverFn onResourceRequest(EventFn callback) { - return this.addEventListener(EVENT_REQUEST, callback); - } - /** * Closes the message stream connection to the server. */ @@ -303,7 +289,7 @@ private void closeStream() { * respond() method. */ @TsInterface - @TsName(namespace = "dh.remotefilesource", name = "ResourceRequest") + @TsName(namespace = "dh.remotefilesource", name = "ResourceRequestEvent") public class ResourceRequestEvent { private final String requestId; private final RemoteFileSourceMetaRequest protoRequest; @@ -328,7 +314,7 @@ public String getResourceName() { * @param content the resource content (string, ArrayBuffer, or typed array), or null if not found */ @JsMethod - public void respond(Object content) { + public void respond(@JsNullable Object content) { // Build RemoteFileSourceMetaResponse proto RemoteFileSourceMetaResponse response = new RemoteFileSourceMetaResponse(); From 07f6b8ef0bf000063417c24a91c10e20bd3bf177 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Thu, 11 Dec 2025 13:41:24 -0600 Subject: [PATCH 20/57] Refactored to use PluginMarker (#DH-20578) --- .../util/RemoteFileSourceClassLoader.java | 4 + .../RemoteFileSourceCommandResolver.java | 22 +- .../RemoteFileSourceMessageStream.java | 360 ++++++++++++++++ .../RemoteFileSourcePlugin.java | 56 +++ .../RemoteFileSourceServicePlugin.java | 387 ------------------ .../deephaven/plugin/type/PluginMarker.java | 46 +++ .../JsRemoteFileSourceService.java | 8 +- .../RemoteFileSourcePluginFetchRequest.java | 19 +- 8 files changed, 501 insertions(+), 401 deletions(-) create mode 100644 plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java create mode 100644 plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java delete mode 100644 plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java create mode 100644 plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java index 4e8d631d2d5..fa8b5bf03bd 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java @@ -41,6 +41,10 @@ public void registerProvider(RemoteFileSourceProvider provider) { providers.add(provider); } + public void unregisterProvider(RemoteFileSourceProvider provider) { + providers.remove(provider); + } + @Override protected URL findResource(String name) { for (RemoteFileSourceProvider provider : providers) { diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java index 4475711928c..af1aabb14db 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java @@ -11,6 +11,7 @@ import io.deephaven.base.verify.Assert; import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; +import io.deephaven.plugin.type.PluginMarker; import io.deephaven.proto.backplane.grpc.RemoteFileSourcePluginFetchRequest; import io.deephaven.proto.backplane.grpc.Ticket; import io.deephaven.proto.util.Exceptions; @@ -70,9 +71,13 @@ private static Any parseOrNull(final ByteString data) { } /** - * Creates and exports a RemoteFileSourceServicePlugin instance based on the fetch request. - * The plugin is exported to the session using the result ticket specified in the request, - * and flight info is returned containing the endpoint for accessing the plugin. + * Exports a PluginMarker singleton based on the fetch request. + * The marker object is exported to the session using the result ticket specified in the request, + * and flight info is returned containing the endpoint for accessing it. + * + * Note: This exports PluginMarker.INSTANCE as a trusted marker. Plugin-specific routing + * is handled by TypedTicket.type in the ConnectRequest phase, which is validated against + * the plugin's name() method. * * @param session the session state for the current request * @param descriptor the flight descriptor containing the command @@ -86,15 +91,16 @@ private static SessionState.ExportObject fetchPlugin(@Nullabl final Ticket resultTicket = request.getResultId(); final boolean hasResultId = !resultTicket.getTicket().isEmpty(); if (!hasResultId) { - throw new StatusRuntimeException(Status.INVALID_ARGUMENT); + throw new StatusRuntimeException(Status.INVALID_ARGUMENT + .withDescription("RemoteFileSourcePluginFetchRequest must contain a valid result_id")); } - final SessionState.ExportBuilder pluginExportBuilder = + final SessionState.ExportBuilder markerExportBuilder = session.newExport(resultTicket, "RemoteFileSourcePluginFetchRequest.resultTicket"); - pluginExportBuilder.require(); + markerExportBuilder.require(); - final SessionState.ExportObject pluginExport = - pluginExportBuilder.submit(RemoteFileSourceServicePlugin::new); + final SessionState.ExportObject markerExport = + markerExportBuilder.submit(() -> PluginMarker.INSTANCE); final Flight.FlightInfo flightInfo = Flight.FlightInfo.newBuilder() .setFlightDescriptor(descriptor) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java new file mode 100644 index 00000000000..ab5232ae9ee --- /dev/null +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java @@ -0,0 +1,360 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.remotefilesource; + +import com.google.protobuf.InvalidProtocolBufferException; +import io.deephaven.internal.log.LoggerFactory; +import io.deephaven.io.logger.Logger; +import io.deephaven.plugin.type.ObjectCommunicationException; +import io.deephaven.plugin.type.ObjectType; +import io.deephaven.proto.backplane.grpc.RemoteFileSourceClientRequest; +import io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaResponse; +import io.deephaven.proto.backplane.grpc.RemoteFileSourceServerRequest; +import io.deephaven.proto.backplane.grpc.SetExecutionContextResponse; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +/** + * Message stream implementation for RemoteFileSource bidirectional communication. + * Each instance represents a file source provider for one client connection and implements + * RemoteFileSourceProvider so it can be registered with the ClassLoader. + * Only one MessageStream can be "active" at a time (determined by the execution context). + * The ClassLoader checks isActive() on each registered provider to find the active one. + */ +public class RemoteFileSourceMessageStream implements ObjectType.MessageStream, io.deephaven.engine.util.RemoteFileSourceProvider { + private static final Logger log = LoggerFactory.getLogger(RemoteFileSourceMessageStream.class); + + /** + * The current execution context containing the active message stream and configuration. + * Null when no execution context is active. + * This is accessed by RemoteFileSourcePlugin.PROVIDER to route resource requests to the + * currently active message stream. + */ + private static volatile RemoteFileSourceExecutionContext executionContext; + + + private final ObjectType.MessageStream connection; + private final Map> pendingRequests = new ConcurrentHashMap<>(); + + public RemoteFileSourceMessageStream(final ObjectType.MessageStream connection) { + this.connection = connection; + // Register this instance as a provider with the ClassLoader + registerWithClassLoader(); + } + + // RemoteFileSourceProvider interface implementation - each instance is a provider + + @Override + public java.util.concurrent.CompletableFuture canSourceResource(String resourceName) { + // Only active if this instance is the currently active message stream + if (!isActive()) { + return java.util.concurrent.CompletableFuture.completedFuture(false); + } + + // Only handle .groovy source files, not compiled .class files + if (!resourceName.endsWith(".groovy")) { + return java.util.concurrent.CompletableFuture.completedFuture(false); + } + + RemoteFileSourceExecutionContext context = executionContext; + if (context == null || context.getActiveMessageStream() != this) { + return java.util.concurrent.CompletableFuture.completedFuture(false); + } + + java.util.List topLevelPackages = context.getTopLevelPackages(); + if (topLevelPackages.isEmpty()) { + return java.util.concurrent.CompletableFuture.completedFuture(false); + } + + String resourcePath = resourceName.replace('\\', '/'); + + for (String topLevelPackage : topLevelPackages) { + String packagePath = topLevelPackage.replace('.', '/'); + if (resourcePath.startsWith(packagePath + "/") || resourcePath.startsWith(packagePath)) { + log.info().append("✅ Can source: ").append(resourceName).endl(); + return java.util.concurrent.CompletableFuture.completedFuture(true); + } + } + + return java.util.concurrent.CompletableFuture.completedFuture(false); + } + + @Override + public java.util.concurrent.CompletableFuture requestResource(String resourceName) { + // Only service requests if this instance is active + if (!isActive()) { + log.warn().append("Request for resource ").append(resourceName) + .append(" on inactive message stream").endl(); + return java.util.concurrent.CompletableFuture.completedFuture(null); + } + + log.info().append("📥 Requesting resource: ").append(resourceName).endl(); + + String requestId = java.util.UUID.randomUUID().toString(); + java.util.concurrent.CompletableFuture future = new java.util.concurrent.CompletableFuture<>(); + pendingRequests.put(requestId, future); + + try { + // Build RemoteFileSourceMetaRequest proto + io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaRequest metaRequest = + io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaRequest.newBuilder() + .setResourceName(resourceName) + .build(); + + // Wrap in RemoteFileSourceServerRequest (server→client) + io.deephaven.proto.backplane.grpc.RemoteFileSourceServerRequest message = + io.deephaven.proto.backplane.grpc.RemoteFileSourceServerRequest.newBuilder() + .setRequestId(requestId) + .setMetaRequest(metaRequest) + .build(); + + java.nio.ByteBuffer buffer = java.nio.ByteBuffer.wrap(message.toByteArray()); + + log.info().append("Sending resource request for: ").append(resourceName) + .append(" with requestId: ").append(requestId).endl(); + + connection.onData(buffer); + } catch (ObjectCommunicationException e) { + future.completeExceptionally(e); + pendingRequests.remove(requestId); + } + + return future; + } + + @Override + public boolean isActive() { + RemoteFileSourceExecutionContext context = executionContext; + return context != null && context.getActiveMessageStream() == this; + } + + // Static methods for execution context management + + /** + * Sets the execution context with the active message stream and top-level packages. + * This should be called when a script execution begins. + * + * @param messageStream the message stream to set as active (must not be null) + * @param packages list of top-level package names to resolve from remote source + * @throws IllegalArgumentException if messageStream is null (use clearExecutionContext() instead) + */ + public static void setExecutionContext(RemoteFileSourceMessageStream messageStream, java.util.List packages) { + if (messageStream == null) { + throw new IllegalArgumentException("messageStream must not be null. Use clearExecutionContext() to clear the context."); + } + + executionContext = new RemoteFileSourceExecutionContext(messageStream, packages); + log.info().append("Set execution context with ") + .append(packages != null ? packages.size() : 0).append(" top-level packages").endl(); + } + + /** + * Clears the execution context. + */ + public static void clearExecutionContext() { + if (executionContext != null) { + executionContext = null; + log.info().append("Cleared execution context").endl(); + } + } + + /** + * Gets the current execution context. + * + * @return the execution context + */ + public static RemoteFileSourceExecutionContext getExecutionContext() { + return executionContext; + } + + // Instance methods for MessageStream implementation + + @Override + public void onData(ByteBuffer payload, Object... references) throws ObjectCommunicationException { + try { + // Parse as RemoteFileSourceClientRequest proto (client→server) + byte[] bytes = new byte[payload.remaining()]; + payload.get(bytes); + RemoteFileSourceClientRequest message = RemoteFileSourceClientRequest.parseFrom(bytes); + + String requestId = message.getRequestId(); + + if (message.hasMetaResponse()) { + // Client is responding to a resource request + RemoteFileSourceMetaResponse response = message.getMetaResponse(); + + CompletableFuture future = pendingRequests.remove(requestId); + if (future != null) { + byte[] content = response.getContent().toByteArray(); + + log.info().append("Received resource response for requestId: ").append(requestId) + .append(", found: ").append(response.getFound()) + .append(", content length: ").append(content.length).endl(); + + if (!response.getError().isEmpty()) { + log.warn().append("Error in response: ").append(response.getError()).endl(); + } + + future.complete(content); + } else { + log.warn().append("Received response for unknown requestId: ").append(requestId).endl(); + } + } else if (message.hasTestCommand()) { + // Client sent a test command + String command = message.getTestCommand(); + log.info().append("Received test command from client: ").append(command).endl(); + + if (command.startsWith("TEST:")) { + String resourceName = command.substring(5).trim(); + log.info().append("Client initiated test for resource: ").append(resourceName).endl(); + testRequestResource(resourceName); + } + } else if (message.hasSetExecutionContext()) { + // Client is requesting this message stream to become active + java.util.List packages = message.getSetExecutionContext().getTopLevelPackagesList(); + setExecutionContext(this, packages); + log.info().append("Client set execution context for this message stream with ") + .append(packages.size()).append(" top-level packages").endl(); + + // Send acknowledgment back to client + SetExecutionContextResponse response = SetExecutionContextResponse.newBuilder() + .setSuccess(true) + .build(); + + RemoteFileSourceServerRequest serverRequest = RemoteFileSourceServerRequest.newBuilder() + .setRequestId(requestId) + .setSetExecutionContextResponse(response) + .build(); + + try { + connection.onData(ByteBuffer.wrap(serverRequest.toByteArray())); + } catch (ObjectCommunicationException e) { + log.error().append("Failed to send execution context acknowledgment: ").append(e).endl(); + } + } else { + log.warn().append("Received unknown message type from client").endl(); + } + } catch (InvalidProtocolBufferException e) { + log.error().append("Failed to parse RemoteFileSourceClientRequest: ").append(e).endl(); + throw new ObjectCommunicationException("Failed to parse message", e); + } + } + + @Override + public void onClose() { + // Unregister this provider from the ClassLoader + unregisterFromClassLoader(); + + // Clear execution context if this was the active stream + RemoteFileSourceExecutionContext context = executionContext; + if (context != null && context.getActiveMessageStream() == this) { + clearExecutionContext(); + } + + // Cancel all pending requests + pendingRequests.values().forEach(future -> future.cancel(true)); + pendingRequests.clear(); + } + + /** + * Register this message stream instance as a provider with the ClassLoader. + */ + private void registerWithClassLoader() { + io.deephaven.engine.util.RemoteFileSourceClassLoader classLoader = + io.deephaven.engine.util.RemoteFileSourceClassLoader.getInstance(); + + if (classLoader != null) { + classLoader.registerProvider(this); + log.info().append("✅ Registered RemoteFileSourceMessageStream provider with ClassLoader").endl(); + } else { + log.warn().append("⚠️ RemoteFileSourceClassLoader not available").endl(); + } + } + + /** + * Unregister this message stream instance from the ClassLoader. + */ + private void unregisterFromClassLoader() { + io.deephaven.engine.util.RemoteFileSourceClassLoader classLoader = + io.deephaven.engine.util.RemoteFileSourceClassLoader.getInstance(); + + if (classLoader != null) { + classLoader.unregisterProvider(this); + log.info().append("🔴 Unregistered RemoteFileSourceMessageStream provider from ClassLoader").endl(); + } + } + + /** + * Test method to request a resource and log the result. This can be called from the server console to test the + * bidirectional communication. + * + * @param resourceName the resource to request + */ + public void testRequestResource(String resourceName) { + log.info().append("Testing resource request for: ").append(resourceName).endl(); + + requestResource(resourceName) + .orTimeout(30, TimeUnit.SECONDS) + .whenComplete((content, error) -> { + if (error != null) { + log.error().append("Error requesting resource ").append(resourceName) + .append(": ").append(error).endl(); + } else { + log.info().append("Successfully received resource ").append(resourceName) + .append(" (").append(content.length).append(" bytes)").endl(); + if (content.length > 0 && content.length < 1000) { + String contentStr = new String(content, StandardCharsets.UTF_8); + log.info().append("Resource content:\n").append(contentStr).endl(); + } + } + }); + } + + /** + * Encapsulates the execution context for remote file source operations. + * This includes the currently active message stream and the top-level packages + * that should be resolved from the remote source. + * This class is immutable - a new instance is created each time the context changes. + */ + public static class RemoteFileSourceExecutionContext { + private final RemoteFileSourceMessageStream activeMessageStream; + private final java.util.List topLevelPackages; + + /** + * Creates a new execution context. + * + * @param activeMessageStream the active message stream + * @param topLevelPackages list of top-level package names to resolve from remote source + */ + public RemoteFileSourceExecutionContext(RemoteFileSourceMessageStream activeMessageStream, + java.util.List topLevelPackages) { + this.activeMessageStream = activeMessageStream; + this.topLevelPackages = topLevelPackages != null ? topLevelPackages : java.util.Collections.emptyList(); + } + + /** + * Gets the currently active message stream. + * + * @return the active message stream + */ + public RemoteFileSourceMessageStream getActiveMessageStream() { + return activeMessageStream; + } + + /** + * Gets the top-level package names that should be resolved from the remote source. + * + * @return a copy of the list of top-level package names + */ + public java.util.List getTopLevelPackages() { + return new java.util.ArrayList<>(topLevelPackages); + } + } +} + diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java new file mode 100644 index 00000000000..35d4d4e87bf --- /dev/null +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java @@ -0,0 +1,56 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.remotefilesource; + +import com.google.auto.service.AutoService; +import io.deephaven.internal.log.LoggerFactory; +import io.deephaven.io.logger.Logger; +import io.deephaven.plugin.type.ObjectType; +import io.deephaven.plugin.type.ObjectTypeBase; +import io.deephaven.plugin.type.ObjectCommunicationException; +import io.deephaven.plugin.type.PluginMarker; + +import java.nio.ByteBuffer; + +/** + * ObjectType plugin for RemoteFileSource. This plugin is registered via @AutoService + * and handles creation of RemoteFileSourceMessageStream connections. + * + * This plugin uses a PluginMarker with a type field instead of instanceof checks, + * allowing it to work across language boundaries (Java/Python). The Flight command + * creates a PluginMarker with type="RemoteFileSource" which this plugin recognizes. + * + * Each RemoteFileSourceMessageStream instance registers itself as a provider with the + * ClassLoader when created and unregisters when closed. The ClassLoader checks isActive() + * on each registered provider to find the currently active one. + */ +@AutoService(ObjectType.class) +public class RemoteFileSourcePlugin extends ObjectTypeBase { + private static final Logger log = LoggerFactory.getLogger(RemoteFileSourcePlugin.class); + + @Override + public String name() { + return "GroovyRemoteFileSourcePlugin"; + } + + @Override + public boolean isType(Object object) { + return object instanceof PluginMarker; + } + + @Override + public MessageStream compatibleClientConnection(Object object, MessageStream connection) + throws ObjectCommunicationException { + if (!isType(object)) { + throw new ObjectCommunicationException("Expected RemoteFileSource marker object, got " + object.getClass()); + } + + connection.onData(ByteBuffer.allocate(0)); + + // Create and return a new message stream for this connection + // All the logic is in the static RemoteFileSourceMessageStream class + return new RemoteFileSourceMessageStream(connection); + } +} + diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java deleted file mode 100644 index ff733394c5e..00000000000 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java +++ /dev/null @@ -1,387 +0,0 @@ -// -// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending -// -package io.deephaven.remotefilesource; - -import com.google.auto.service.AutoService; -import com.google.protobuf.InvalidProtocolBufferException; -import io.deephaven.internal.log.LoggerFactory; -import io.deephaven.io.logger.Logger; -import io.deephaven.plugin.type.ObjectType; -import io.deephaven.plugin.type.ObjectTypeBase; -import io.deephaven.plugin.type.ObjectCommunicationException; -import io.deephaven.proto.backplane.grpc.RemoteFileSourceClientRequest; -import io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaRequest; -import io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaResponse; -import io.deephaven.proto.backplane.grpc.RemoteFileSourceServerRequest; -import io.deephaven.proto.backplane.grpc.SetExecutionContextResponse; - -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; - -@AutoService(ObjectType.class) -public class RemoteFileSourceServicePlugin extends ObjectTypeBase implements io.deephaven.engine.util.RemoteFileSourceProvider { - private static final Logger log = LoggerFactory.getLogger(RemoteFileSourceServicePlugin.class); - - /** - * The current execution context containing the active message stream and configuration. - * Null when no execution context is active. - */ - private static volatile RemoteFileSourceExecutionContext executionContext; - - private volatile RemoteFileSourceMessageStream messageStream; - - public RemoteFileSourceServicePlugin() { - log.info().append("🎯 RemoteFileSourceServicePlugin constructor called").endl(); - // Register eagerly with the class loader - registerWithClassLoader(); - } - - // RemoteFileSourceProvider interface implementation - delegates to active message stream - - @Override - public CompletableFuture canSourceResource(String resourceName) { - // Only handle .groovy source files, not compiled .class files - if (!resourceName.endsWith(".groovy")) { - return CompletableFuture.completedFuture(false); - } - - RemoteFileSourceExecutionContext context = executionContext; - if (context == null) { - return CompletableFuture.completedFuture(false); - } - - java.util.List topLevelPackages = context.getTopLevelPackages(); - if (topLevelPackages.isEmpty()) { - return CompletableFuture.completedFuture(false); - } - - String resourcePath = resourceName.replace('\\', '/'); - - for (String topLevelPackage : topLevelPackages) { - String packagePath = topLevelPackage.replace('.', '/'); - if (resourcePath.startsWith(packagePath + "/") || resourcePath.startsWith(packagePath)) { - log.info().append("✅ Can source: ").append(resourceName).endl(); - return CompletableFuture.completedFuture(true); - } - } - - return CompletableFuture.completedFuture(false); - } - - @Override - public CompletableFuture requestResource(String resourceName) { - log.info().append("📥 Requesting resource: ").append(resourceName).endl(); - - RemoteFileSourceExecutionContext context = executionContext; - if (context == null) { - log.warn().append("No execution context when requesting resource").endl(); - return CompletableFuture.completedFuture(null); - } - - return context.getActiveMessageStream().requestResource(resourceName); - } - - @Override - public boolean isActive() { - return executionContext != null; - } - - /** - * Sets the execution context with the active message stream and top-level packages. - * This should be called when a script execution begins. - * The plugin (which is registered with the ClassLoader) will route requests to this message stream. - * - * @param messageStream the message stream to set as active (must not be null) - * @param packages list of top-level package names to resolve from remote source - * @throws IllegalArgumentException if messageStream is null (use clearExecutionContext() instead) - */ - public static void setExecutionContext(RemoteFileSourceMessageStream messageStream, java.util.List packages) { - if (messageStream == null) { - throw new IllegalArgumentException("messageStream must not be null. Use clearExecutionContext() to clear the context."); - } - - // Set new context - the plugin will automatically route to this message stream - executionContext = new RemoteFileSourceExecutionContext(messageStream, packages); - log.info().append("Set execution context with ") - .append(packages != null ? packages.size() : 0).append(" top-level packages").endl(); - } - - /** - * Clears the execution context. - */ - public static void clearExecutionContext() { - if (executionContext != null) { - executionContext = null; - log.info().append("Cleared execution context").endl(); - } - } - - /** - * Register this plugin instance with the RemoteFileSourceClassLoader instance. - * Called once during plugin construction. - */ - private void registerWithClassLoader() { - io.deephaven.engine.util.RemoteFileSourceClassLoader classLoader = - io.deephaven.engine.util.RemoteFileSourceClassLoader.getInstance(); - - if (classLoader != null) { - classLoader.registerProvider(this); - log.info().append("✅ Registered RemoteFileSourceServicePlugin with RemoteFileSourceClassLoader").endl(); - } else { - log.warn().append("⚠️ RemoteFileSourceClassLoader instance not found - plugin not registered").endl(); - } - } - - - /** - * Gets the current execution context. - * - * @return the execution context - */ - public static RemoteFileSourceExecutionContext getExecutionContext() { - return executionContext; - } - - @Override - public String name() { - return "RemoteFileSourceService"; - } - - @Override - public boolean isType(Object object) { - return object instanceof RemoteFileSourceServicePlugin; - } - - @Override - public MessageStream compatibleClientConnection(Object object, MessageStream connection) - throws ObjectCommunicationException { - connection.onData(ByteBuffer.allocate(0)); - messageStream = new RemoteFileSourceMessageStream(connection); - return messageStream; - } - - /** - * Test method to trigger a resource request from the server to the client. Can be called from the console to test - * bidirectional communication. Usage from console: - * 
-     * service = remote_file_source_service  # The plugin instance
-     * service.testRequestResource("com/example/MyClass.java")
-     *
- * @param resourceName the resource to request from the client - */ - public void testRequestResource(String resourceName) { - if (messageStream == null) { - log.error().append("MessageStream not connected. Please connect a client first.").endl(); - return; - } - messageStream.testRequestResource(resourceName); - } - - /** - * A message stream for the RemoteFileSourceService. - */ - public static class RemoteFileSourceMessageStream implements MessageStream { - private final MessageStream connection; - private final Map> pendingRequests = new ConcurrentHashMap<>(); - - public RemoteFileSourceMessageStream(final MessageStream connection) { - this.connection = connection; - } - - @Override - public void onData(ByteBuffer payload, Object... references) throws ObjectCommunicationException { - try { - // Parse as RemoteFileSourceClientRequest proto (client→server) - byte[] bytes = new byte[payload.remaining()]; - payload.get(bytes); - RemoteFileSourceClientRequest message = RemoteFileSourceClientRequest.parseFrom(bytes); - - String requestId = message.getRequestId(); - - if (message.hasMetaResponse()) { - // Client is responding to a resource request - RemoteFileSourceMetaResponse response = message.getMetaResponse(); - - CompletableFuture future = pendingRequests.remove(requestId); - if (future != null) { - byte[] content = response.getContent().toByteArray(); - - log.info().append("Received resource response for requestId: ").append(requestId) - .append(", found: ").append(response.getFound()) - .append(", content length: ").append(content.length).endl(); - - if (!response.getError().isEmpty()) { - log.warn().append("Error in response: ").append(response.getError()).endl(); - } - - // Complete the future - the caller will log the content if needed - future.complete(content); - } else { - log.warn().append("Received response for unknown requestId: ").append(requestId).endl(); - } - } else if (message.hasTestCommand()) { - // Client sent a test command - String command = message.getTestCommand(); - log.info().append("Received test command from client: ").append(command).endl(); - - if (command.startsWith("TEST:")) { - String resourceName = command.substring(5).trim(); - log.info().append("Client initiated test for resource: ").append(resourceName).endl(); - testRequestResource(resourceName); - } - } else if (message.hasSetExecutionContext()) { - // Client is requesting this message stream to become active - java.util.List packages = message.getSetExecutionContext().getTopLevelPackagesList(); - setExecutionContext(this, packages); - log.info().append("Client set execution context for this message stream with ") - .append(packages.size()).append(" top-level packages").endl(); - - // Send acknowledgment back to client - SetExecutionContextResponse response = SetExecutionContextResponse.newBuilder() - .setSuccess(true) - .build(); - - RemoteFileSourceServerRequest serverRequest = RemoteFileSourceServerRequest.newBuilder() - .setRequestId(requestId) - .setSetExecutionContextResponse(response) - .build(); - - try { - connection.onData(ByteBuffer.wrap(serverRequest.toByteArray())); - } catch (ObjectCommunicationException e) { - log.error().append("Failed to send execution context acknowledgment: ").append(e).endl(); - } - } else { - log.warn().append("Received unknown message type from client").endl(); - } - } catch (InvalidProtocolBufferException e) { - log.error().append("Failed to parse RemoteFileSourceClientRequest: ").append(e).endl(); - throw new ObjectCommunicationException("Failed to parse message", e); - } - } - - @Override - public void onClose() { - // Clear execution context if this was the active stream - RemoteFileSourceExecutionContext context = executionContext; - if (context != null && context.getActiveMessageStream() == this) { - clearExecutionContext(); - } - - // Cancel all pending requests - pendingRequests.values().forEach(future -> future.cancel(true)); - pendingRequests.clear(); - } - - /** - * Request a resource from the client. - * - * @param resourceName the name/path of the resource to request - * @return a future that completes with the resource content, or empty array if not found - */ - public CompletableFuture requestResource(String resourceName) { - String requestId = UUID.randomUUID().toString(); - CompletableFuture future = new CompletableFuture<>(); - pendingRequests.put(requestId, future); - - try { - // Build RemoteFileSourceMetaRequest proto - RemoteFileSourceMetaRequest metaRequest = RemoteFileSourceMetaRequest.newBuilder() - .setResourceName(resourceName) - .build(); - - // Wrap in RemoteFileSourceServerRequest (server→client) - RemoteFileSourceServerRequest message = RemoteFileSourceServerRequest.newBuilder() - .setRequestId(requestId) - .setMetaRequest(metaRequest) - .build(); - - ByteBuffer buffer = ByteBuffer.wrap(message.toByteArray()); - - log.info().append("Sending resource request for: ").append(resourceName) - .append(" with requestId: ").append(requestId).endl(); - - connection.onData(buffer); - } catch (ObjectCommunicationException e) { - future.completeExceptionally(e); - pendingRequests.remove(requestId); - } - - return future; - } - - - /** - * Test method to request a resource and log the result. This can be called from the server console to test the - * bidirectional communication. - * - * @param resourceName the resource to request - */ - public void testRequestResource(String resourceName) { - log.info().append("Testing resource request for: ").append(resourceName).endl(); - - requestResource(resourceName) - .orTimeout(30, TimeUnit.SECONDS) - .whenComplete((content, error) -> { - if (error != null) { - log.error().append("Error requesting resource ").append(resourceName) - .append(": ").append(error).endl(); - } else { - log.info().append("Successfully received resource ").append(resourceName) - .append(" (").append(content.length).append(" bytes)").endl(); - if (content.length > 0 && content.length < 1000) { - String contentStr = new String(content, StandardCharsets.UTF_8); - log.info().append("Resource content:\n").append(contentStr).endl(); - } - } - }); - } - } - - /** - * Encapsulates the execution context for remote file source operations. - * This includes the currently active message stream and the top-level packages - * that should be resolved from the remote source. - * This class is immutable - a new instance is created each time the context changes. - */ - public static class RemoteFileSourceExecutionContext { - private final RemoteFileSourceMessageStream activeMessageStream; - private final java.util.List topLevelPackages; - - /** - * Creates a new execution context. - * - * @param activeMessageStream the active message stream - * @param topLevelPackages list of top-level package names to resolve from remote source - */ - public RemoteFileSourceExecutionContext(RemoteFileSourceMessageStream activeMessageStream, - java.util.List topLevelPackages) { - this.activeMessageStream = activeMessageStream; - this.topLevelPackages = topLevelPackages != null ? topLevelPackages : java.util.Collections.emptyList(); - } - - /** - * Gets the currently active message stream. - * - * @return the active message stream - */ - public RemoteFileSourceMessageStream getActiveMessageStream() { - return activeMessageStream; - } - - /** - * Gets the top-level package names that should be resolved from the remote source. - * - * @return a copy of the list of top-level package names - */ - public java.util.List getTopLevelPackages() { - return new java.util.ArrayList<>(topLevelPackages); - } - } -} diff --git a/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java b/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java new file mode 100644 index 00000000000..48d90cdc7d4 --- /dev/null +++ b/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java @@ -0,0 +1,46 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.plugin.type; + +/** + * A generic marker object for plugin exports that can be shared across multiple plugin types. + * The actual plugin routing is handled by TypedTicket.type, which is validated against + * ObjectType.name() during the ConnectRequest phase. This marker simply indicates "this + * exported object is a plugin placeholder" rather than containing actual plugin-specific data. + */ +public class PluginMarker { + /** + * Static type identifier for PluginMarker objects. + * Uses the fully qualified class name as a standard cross-language identifier. + */ + public static final String TYPE_ID = "io.deephaven.plugin.type.PluginMarker"; + + /** + * Singleton instance for all plugin marker exports. + * Since plugin-specific routing is handled by TypedTicket.type, we don't need + * per-plugin marker instances. + */ + public static final PluginMarker INSTANCE = new PluginMarker(); + + /** + * Private constructor - use INSTANCE singleton. + */ + private PluginMarker() { + } + + /** + * Gets the static type identifier for all PluginMarker objects. + * + * @return the static type identifier + */ + public String getTypeId() { + return TYPE_ID; + } + + @Override + public String toString() { + return "PluginMarker{typeId='" + TYPE_ID + "'}"; + } +} + diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index 0aba6fe8614..55e55177293 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -108,7 +108,10 @@ public static Promise fetchPlugin(@TsTypeRef(Object.c return Callbacks.grpcUnaryPromise( c -> connection.flightServiceClient().getFlightInfo(descriptor, connection.metadata(), c::apply)) .then(flightInfo -> { - // The first endpoint should contain the ticket for the plugin instance + // The first endpoint contains the ticket for the plugin instance. + // This is the standard Flight pattern: we passed resultTicket in the request, + // the server exported the service to that ticket, and returned a FlightInfo + // with an endpoint containing that same ticket for us to use. if (flightInfo.getEndpointList().length > 0) { // Get the Arrow Flight ticket from the endpoint io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.Ticket flightTicket = @@ -119,9 +122,10 @@ public static Promise fetchPlugin(@TsTypeRef(Object.c dhTicket.setTicket(flightTicket.getTicket_asU8()); // Create a TypedTicket for the plugin instance + // The type must match RemoteFileSourcePlugin.name() TypedTicket typedTicket = new TypedTicket(); typedTicket.setTicket(dhTicket); - typedTicket.setType("RemoteFileSourceService"); + typedTicket.setType("GroovyRemoteFileSourcePlugin"); // Create a new service instance with the typed ticket and connect to it JsRemoteFileSourceService service = new JsRemoteFileSourceService(connection, typedTicket); diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java index 203f56eb24e..224a65446f8 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java @@ -85,9 +85,15 @@ static RemoteFileSourcePluginFetchRequest.ToObjectReturnType create() { @JsProperty RemoteFileSourcePluginFetchRequest.ToObjectReturnType.ResultIdFieldType getResultId(); + @JsProperty + String getPluginType(); + @JsProperty void setResultId( RemoteFileSourcePluginFetchRequest.ToObjectReturnType.ResultIdFieldType resultId); + + @JsProperty + void setPluginType(String pluginType); } @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) @@ -158,9 +164,15 @@ static RemoteFileSourcePluginFetchRequest.ToObjectReturnType0 create() { @JsProperty RemoteFileSourcePluginFetchRequest.ToObjectReturnType0.ResultIdFieldType getResultId(); + @JsProperty + String getPluginType(); + @JsProperty void setResultId( RemoteFileSourcePluginFetchRequest.ToObjectReturnType0.ResultIdFieldType resultId); + + @JsProperty + void setPluginType(String pluginType); } public static native RemoteFileSourcePluginFetchRequest deserializeBinary(Uint8Array bytes); @@ -176,11 +188,9 @@ public static native RemoteFileSourcePluginFetchRequest.ToObjectReturnType toObj public native void clearResultId(); - public native void clearClientSessionId(); - public native Ticket getResultId(); - public native String getClientSessionId(); + public native String getPluginType(); public native boolean hasResultId(); @@ -190,7 +200,8 @@ public static native RemoteFileSourcePluginFetchRequest.ToObjectReturnType toObj public native void setResultId(Ticket value); - public native void setClientSessionId(String value); + + public native void setPluginType(String value); public native RemoteFileSourcePluginFetchRequest.ToObjectReturnType0 toObject(); From 3d7ebc7605d05ccaa67b11a9be1752033deb181e Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Thu, 11 Dec 2025 16:35:08 -0600 Subject: [PATCH 21/57] Re-using JsWidget for message stream (#DH-20578) --- .../RemoteFileSourceCommandResolver.java | 2 +- .../RemoteFileSourcePlugin.java | 2 +- .../JsRemoteFileSourceService.java | 153 ++++++------------ 3 files changed, 49 insertions(+), 108 deletions(-) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java index af1aabb14db..5f0cfd6556c 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java @@ -97,7 +97,7 @@ private static SessionState.ExportObject fetchPlugin(@Nullabl final SessionState.ExportBuilder markerExportBuilder = session.newExport(resultTicket, "RemoteFileSourcePluginFetchRequest.resultTicket"); - markerExportBuilder.require(); +// markerExportBuilder.require(); final SessionState.ExportObject markerExport = markerExportBuilder.submit(() -> PluginMarker.INSTANCE); diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java index 35d4d4e87bf..3c3e26048ed 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java @@ -31,7 +31,7 @@ public class RemoteFileSourcePlugin extends ObjectTypeBase { @Override public String name() { - return "GroovyRemoteFileSourcePlugin"; + return "DeephavenGroovyRemoteFileSourcePlugin"; } @Override diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index 55e55177293..88db3b87808 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -11,10 +11,6 @@ import elemental2.promise.Promise; import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.FlightDescriptor; import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.FlightInfo; -import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.object_pb.ClientData; -import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.object_pb.ConnectRequest; -import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.object_pb.StreamRequest; -import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.object_pb.StreamResponse; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceClientRequest; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceMetaRequest; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceMetaResponse; @@ -25,9 +21,11 @@ import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.Ticket; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.TypedTicket; import io.deephaven.web.client.api.Callbacks; +import io.deephaven.web.client.api.event.Event; import io.deephaven.web.client.api.WorkerConnection; -import io.deephaven.web.client.api.barrage.stream.BiDiStream; import io.deephaven.web.client.api.event.HasEventHandling; +import io.deephaven.web.client.api.widget.JsWidget; +import io.deephaven.web.client.api.widget.WidgetMessageDetails; import jsinterop.annotations.JsIgnore; import jsinterop.annotations.JsMethod; import jsinterop.annotations.JsNullable; @@ -38,7 +36,6 @@ import java.util.HashMap; import java.util.Map; -import java.util.function.Supplier; /** @@ -48,32 +45,17 @@ @JsType(namespace = "dh.remotefilesource", name = "RemoteFileSourceService") public class JsRemoteFileSourceService extends HasEventHandling { public static final String EVENT_MESSAGE = "message"; - public static final String EVENT_CLOSE = "close"; public static final String EVENT_REQUEST = "request"; - private final TypedTicket typedTicket; - - private final Supplier> streamFactory; - private BiDiStream messageStream; - - private boolean hasFetched; + private final JsWidget widget; // Track pending setExecutionContext requests private final Map> pendingSetExecutionContextRequests = new HashMap<>(); private int requestIdCounter = 0; @JsIgnore - private JsRemoteFileSourceService(WorkerConnection connection, TypedTicket typedTicket) { - this.typedTicket = typedTicket; - this.hasFetched = false; - - // Set up the message stream factory - BiDiStream.Factory factory = connection.streamFactory(); - this.streamFactory = () -> factory.create( - connection.objectServiceClient()::messageStream, - (first, headers) -> connection.objectServiceClient().openMessageStream(first, headers), - (next, headers, c) -> connection.objectServiceClient().nextMessageStream(next, headers, c::apply), - new StreamRequest()); + private JsRemoteFileSourceService(JsWidget widget) { + this.widget = widget; } /** @@ -82,7 +64,7 @@ private JsRemoteFileSourceService(WorkerConnection connection, TypedTicket typed * @param connection the worker connection to use for communication * @return a promise that resolves to a RemoteFileSourceService instance with an active message stream */ - @JsMethod + @JsIgnore public static Promise fetchPlugin(@TsTypeRef(Object.class) WorkerConnection connection) { // Create a new export ticket for the result Ticket resultTicket = connection.getTickets().newExportTicket(); @@ -125,10 +107,11 @@ public static Promise fetchPlugin(@TsTypeRef(Object.c // The type must match RemoteFileSourcePlugin.name() TypedTicket typedTicket = new TypedTicket(); typedTicket.setTicket(dhTicket); - typedTicket.setType("GroovyRemoteFileSourcePlugin"); + typedTicket.setType("DeephavenGroovyRemoteFileSourcePlugin"); + + JsWidget widget = new JsWidget(connection, typedTicket); - // Create a new service instance with the typed ticket and connect to it - JsRemoteFileSourceService service = new JsRemoteFileSourceService(connection, typedTicket); + JsRemoteFileSourceService service = new JsRemoteFileSourceService(widget); return service.connect(); } else { return Promise.reject("No endpoints returned from RemoteFileSource plugin fetch"); @@ -143,70 +126,40 @@ public static Promise fetchPlugin(@TsTypeRef(Object.c */ @JsIgnore private Promise connect() { - if (messageStream != null) { - messageStream.end(); - } - - return new Promise<>((resolve, reject) -> { - messageStream = streamFactory.get(); - - messageStream.onData(res -> { - if (!hasFetched) { - hasFetched = true; - resolve.onInvoke(this); - } else { - // Parse the message as RemoteFileSourceServerRequest proto (server→client) - Uint8Array payload = res.getData().getPayload_asU8(); - - try { - RemoteFileSourceServerRequest message = - RemoteFileSourceServerRequest.deserializeBinary(payload); - - // Check which message type it is - if (message.hasMetaRequest()) { - // Server is requesting a resource from the client - RemoteFileSourceMetaRequest request = message.getMetaRequest(); - - // Fire request event (include request_id from wrapper) - DomGlobal.setTimeout(ignore -> fireEvent(EVENT_REQUEST, - new ResourceRequestEvent(message.getRequestId(), request)), 0); - } else if (message.hasSetExecutionContextResponse()) { - // Server acknowledged execution context - String requestId = message.getRequestId(); - Promise.PromiseExecutorCallbackFn.ResolveCallbackFn resolveCallback = - pendingSetExecutionContextRequests.remove(requestId); - if (resolveCallback != null) { - SetExecutionContextResponse response = message.getSetExecutionContextResponse(); - resolveCallback.onInvoke(response.getSuccess()); - } - } else { - // Unknown message type - DomGlobal.setTimeout(ignore -> fireEvent(EVENT_MESSAGE, res.getData()), 0); - } - } catch (Exception e) { - // Failed to parse as proto, fire generic message event - DomGlobal.setTimeout(ignore -> fireEvent(EVENT_MESSAGE, res.getData()), 0); + widget.addEventListener("message", (Event event) -> { + // Parse the message as RemoteFileSourceServerRequest proto (server→client) + Uint8Array payload = event.getDetail().getDataAsU8(); + + try { + RemoteFileSourceServerRequest message = + RemoteFileSourceServerRequest.deserializeBinary(payload); + + if (message.hasMetaRequest()) { + // If server has requested a resource from the client, fire request event + RemoteFileSourceMetaRequest request = message.getMetaRequest(); + + DomGlobal.setTimeout(ignore -> fireEvent(EVENT_REQUEST, + new ResourceRequestEvent(message.getRequestId(), request)), 0); + } else if (message.hasSetExecutionContextResponse()) { + // Server acknowledged execution context was set + String requestId = message.getRequestId(); + Promise.PromiseExecutorCallbackFn.ResolveCallbackFn resolveCallback = + pendingSetExecutionContextRequests.remove(requestId); + if (resolveCallback != null) { + SetExecutionContextResponse response = message.getSetExecutionContextResponse(); + resolveCallback.onInvoke(response.getSuccess()); } + } else { + // Unknown message type + DomGlobal.setTimeout(ignore -> fireEvent(EVENT_MESSAGE, event.getDetail()), 0); } - }); - - messageStream.onStatus(status -> { - if (!status.isOk()) { - reject.onInvoke(status.getDetails()); - } - DomGlobal.setTimeout(ignore -> fireEvent(EVENT_CLOSE), 0); - closeStream(); - }); - - messageStream.onEnd(status -> closeStream()); - - // First message establishes a connection w/ the plugin object instance we're talking to - StreamRequest req = new StreamRequest(); - ConnectRequest data = new ConnectRequest(); - data.setSourceId(typedTicket); - req.setConnect(data); - messageStream.send(req); + } catch (Exception e) { + // Failed to parse as proto, fire generic message event + DomGlobal.setTimeout(ignore -> fireEvent(EVENT_MESSAGE, event.getDetail()), 0); + } }); + + return widget.refetch().then(w -> Promise.resolve(this)); } /** @@ -261,15 +214,11 @@ public Promise setExecutionContext(@JsOptional String[] topLevelPackage */ @JsIgnore private void sendClientRequest(RemoteFileSourceClientRequest clientRequest) { - if (messageStream == null) { - throw new IllegalStateException("Message stream not connected"); - } + // Serialize the protobuf message to bytes + Uint8Array messageBytes = clientRequest.serializeBinary(); - StreamRequest req = new StreamRequest(); - ClientData clientData = new ClientData(); - clientData.setPayload(clientRequest.serializeBinary()); - req.setData(clientData); - messageStream.send(req); + // Send as Uint8Array (which is an ArrayBufferView, compatible with MessageUnion) + widget.sendMessage(Js.uncheckedCast(messageBytes), null); } /** @@ -277,15 +226,7 @@ private void sendClientRequest(RemoteFileSourceClientRequest clientRequest) { */ @JsMethod public void close() { - closeStream(); - } - - @JsIgnore - private void closeStream() { - if (messageStream != null) { - messageStream.end(); - messageStream = null; - } + widget.close(); } /** From eea72141be981a0a36e7dcc1a4380dd7de3e8a58 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Thu, 11 Dec 2025 17:55:34 -0600 Subject: [PATCH 22/57] Added pluginType field to PluginMarker (#DH-20578) --- .../RemoteFileSourceCommandResolver.java | 10 +++- .../RemoteFileSourcePlugin.java | 7 ++- .../deephaven/plugin/type/PluginMarker.java | 48 +++++++++++++++---- .../proto/remotefilesource.proto | 3 ++ .../JsRemoteFileSourceService.java | 5 +- 5 files changed, 60 insertions(+), 13 deletions(-) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java index 5f0cfd6556c..f86d54d0560 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java @@ -95,12 +95,20 @@ private static SessionState.ExportObject fetchPlugin(@Nullabl .withDescription("RemoteFileSourcePluginFetchRequest must contain a valid result_id")); } + final String pluginType = request.getPluginType(); + if (pluginType == null || pluginType.isEmpty()) { + throw new StatusRuntimeException(Status.INVALID_ARGUMENT + .withDescription("RemoteFileSourcePluginFetchRequest must contain a valid plugin_type")); + } + final SessionState.ExportBuilder markerExportBuilder = session.newExport(resultTicket, "RemoteFileSourcePluginFetchRequest.resultTicket"); // markerExportBuilder.require(); + // Get singleton marker for this plugin type + // This ensures isType() routing works correctly when multiple plugins use PluginMarker final SessionState.ExportObject markerExport = - markerExportBuilder.submit(() -> PluginMarker.INSTANCE); + markerExportBuilder.submit(() -> PluginMarker.forPluginType(pluginType)); final Flight.FlightInfo flightInfo = Flight.FlightInfo.newBuilder() .setFlightDescriptor(descriptor) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java index 3c3e26048ed..f76746a0e1c 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java @@ -36,7 +36,12 @@ public String name() { @Override public boolean isType(Object object) { - return object instanceof PluginMarker; + // We need to check the pluginType + if (object instanceof PluginMarker) { + PluginMarker marker = (PluginMarker) object; + return name().equals(marker.getPluginType()); + } + return false; } @Override diff --git a/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java b/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java index 48d90cdc7d4..b84ff58dbb9 100644 --- a/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java +++ b/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java @@ -5,9 +5,13 @@ /** * A generic marker object for plugin exports that can be shared across multiple plugin types. - * The actual plugin routing is handled by TypedTicket.type, which is validated against - * ObjectType.name() during the ConnectRequest phase. This marker simply indicates "this - * exported object is a plugin placeholder" rather than containing actual plugin-specific data. + * + * IMPORTANT: The pluginType field is required because ObjectTypeLookup.findObjectType() + * returns the FIRST plugin where isType() returns true. Without plugin-specific identification + * in isType(), multiple plugins using PluginMarker would conflict, and whichever is registered + * first would intercept all PluginMarker instances. + * + * This class uses a singleton pattern - one instance per pluginType. */ public class PluginMarker { /** @@ -16,17 +20,31 @@ public class PluginMarker { */ public static final String TYPE_ID = "io.deephaven.plugin.type.PluginMarker"; + private static final java.util.Map INSTANCES = new java.util.concurrent.ConcurrentHashMap<>(); + + private final String pluginType; + /** - * Singleton instance for all plugin marker exports. - * Since plugin-specific routing is handled by TypedTicket.type, we don't need - * per-plugin marker instances. + * Private constructor - use forPluginType() to get singleton instances. + * + * @param pluginType the plugin type identifier (should match the plugin's name() method) */ - public static final PluginMarker INSTANCE = new PluginMarker(); + private PluginMarker(String pluginType) { + this.pluginType = pluginType; + } /** - * Private constructor - use INSTANCE singleton. + * Gets the singleton PluginMarker instance for the specified plugin type. + * + * @param pluginType the plugin type identifier (should match the plugin's name() method) + * @return the singleton PluginMarker for this plugin type + * @throws IllegalArgumentException if pluginType is null or empty */ - private PluginMarker() { + public static PluginMarker forPluginType(String pluginType) { + if (pluginType == null || pluginType.isEmpty()) { + throw new IllegalArgumentException("pluginType cannot be null or empty"); + } + return INSTANCES.computeIfAbsent(pluginType, PluginMarker::new); } /** @@ -38,9 +56,19 @@ public String getTypeId() { return TYPE_ID; } + /** + * Gets the plugin type this marker is intended for. + * This should match the ObjectType.name() of the target plugin. + * + * @return the plugin type identifier + */ + public String getPluginType() { + return pluginType; + } + @Override public String toString() { - return "PluginMarker{typeId='" + TYPE_ID + "'}"; + return "PluginMarker{typeId='" + TYPE_ID + "', pluginType='" + pluginType + "'}"; } } diff --git a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto index 5fef67186f4..12d9aff1831 100644 --- a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto +++ b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto @@ -76,4 +76,7 @@ message SetExecutionContextResponse { // Fetch the remote file source plugin into the specified ticket (Flight command, not MessageStream) message RemoteFileSourcePluginFetchRequest { io.deephaven.proto.backplane.grpc.Ticket result_id = 1; + + // The plugin type identifier to create the PluginMarker for. + string plugin_type = 2; } \ No newline at end of file diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index 88db3b87808..baf9b21c3cb 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -66,12 +66,15 @@ private JsRemoteFileSourceService(JsWidget widget) { */ @JsIgnore public static Promise fetchPlugin(@TsTypeRef(Object.class) WorkerConnection connection) { + String pluginType = "DeephavenGroovyRemoteFileSourcePlugin"; + // Create a new export ticket for the result Ticket resultTicket = connection.getTickets().newExportTicket(); // Create the fetch request RemoteFileSourcePluginFetchRequest fetchRequest = new RemoteFileSourcePluginFetchRequest(); fetchRequest.setResultId(resultTicket); + fetchRequest.setPluginType(pluginType); // Serialize the request to bytes Uint8Array innerRequestBytes = fetchRequest.serializeBinary(); @@ -107,7 +110,7 @@ public static Promise fetchPlugin(@TsTypeRef(Object.c // The type must match RemoteFileSourcePlugin.name() TypedTicket typedTicket = new TypedTicket(); typedTicket.setTicket(dhTicket); - typedTicket.setType("DeephavenGroovyRemoteFileSourcePlugin"); + typedTicket.setType(pluginType); JsWidget widget = new JsWidget(connection, typedTicket); From bc1a90b3fa504aa0993c6a9c49a7b6bbb2cf984f Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Fri, 12 Dec 2025 12:52:14 -0600 Subject: [PATCH 23/57] Removed redundant field and renamed pluginType to pluginName (#DH-20578) --- .../RemoteFileSourceCommandResolver.java | 11 ++-- .../RemoteFileSourcePlugin.java | 3 +- .../deephaven/plugin/type/PluginMarker.java | 55 +++++++------------ .../proto/remotefilesource.proto | 4 +- .../JsRemoteFileSourceService.java | 6 +- .../RemoteFileSourcePluginFetchRequest.java | 13 ++--- 6 files changed, 37 insertions(+), 55 deletions(-) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java index f86d54d0560..c1d973609af 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java @@ -95,20 +95,19 @@ private static SessionState.ExportObject fetchPlugin(@Nullabl .withDescription("RemoteFileSourcePluginFetchRequest must contain a valid result_id")); } - final String pluginType = request.getPluginType(); - if (pluginType == null || pluginType.isEmpty()) { + final String pluginName = request.getPluginName(); + if (pluginName.isEmpty()) { throw new StatusRuntimeException(Status.INVALID_ARGUMENT - .withDescription("RemoteFileSourcePluginFetchRequest must contain a valid plugin_type")); + .withDescription("RemoteFileSourcePluginFetchRequest must contain a valid plugin_name")); } final SessionState.ExportBuilder markerExportBuilder = session.newExport(resultTicket, "RemoteFileSourcePluginFetchRequest.resultTicket"); -// markerExportBuilder.require(); - // Get singleton marker for this plugin type + // Get singleton marker for this plugin name // This ensures isType() routing works correctly when multiple plugins use PluginMarker final SessionState.ExportObject markerExport = - markerExportBuilder.submit(() -> PluginMarker.forPluginType(pluginType)); + markerExportBuilder.submit(() -> PluginMarker.forPluginName(pluginName)); final Flight.FlightInfo flightInfo = Flight.FlightInfo.newBuilder() .setFlightDescriptor(descriptor) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java index f76746a0e1c..09af2b3430f 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java @@ -36,10 +36,9 @@ public String name() { @Override public boolean isType(Object object) { - // We need to check the pluginType if (object instanceof PluginMarker) { PluginMarker marker = (PluginMarker) object; - return name().equals(marker.getPluginType()); + return name().equals(marker.getPluginName()); } return false; } diff --git a/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java b/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java index b84ff58dbb9..be32f2227fb 100644 --- a/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java +++ b/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java @@ -6,69 +6,54 @@ /** * A generic marker object for plugin exports that can be shared across multiple plugin types. * - * IMPORTANT: The pluginType field is required because ObjectTypeLookup.findObjectType() + * IMPORTANT: The pluginName field is required because ObjectTypeLookup.findObjectType() * returns the FIRST plugin where isType() returns true. Without plugin-specific identification * in isType(), multiple plugins using PluginMarker would conflict, and whichever is registered * first would intercept all PluginMarker instances. * - * This class uses a singleton pattern - one instance per pluginType. + * This class uses a singleton pattern - one instance per pluginName. */ public class PluginMarker { - /** - * Static type identifier for PluginMarker objects. - * Uses the fully qualified class name as a standard cross-language identifier. - */ - public static final String TYPE_ID = "io.deephaven.plugin.type.PluginMarker"; - private static final java.util.Map INSTANCES = new java.util.concurrent.ConcurrentHashMap<>(); - private final String pluginType; + private final String pluginName; /** - * Private constructor - use forPluginType() to get singleton instances. + * Private constructor - use forPluginName() to get singleton instances. * - * @param pluginType the plugin type identifier (should match the plugin's name() method) + * @param pluginName the plugin name identifier (should match the plugin's name() method) */ - private PluginMarker(String pluginType) { - this.pluginType = pluginType; + private PluginMarker(String pluginName) { + this.pluginName = pluginName; } /** - * Gets the singleton PluginMarker instance for the specified plugin type. + * Gets the singleton PluginMarker instance for the specified plugin name. * - * @param pluginType the plugin type identifier (should match the plugin's name() method) - * @return the singleton PluginMarker for this plugin type - * @throws IllegalArgumentException if pluginType is null or empty + * @param pluginName the plugin name identifier (should match the plugin's name() method) + * @return the singleton PluginMarker for this plugin name + * @throws IllegalArgumentException if pluginName is null or empty */ - public static PluginMarker forPluginType(String pluginType) { - if (pluginType == null || pluginType.isEmpty()) { - throw new IllegalArgumentException("pluginType cannot be null or empty"); + public static PluginMarker forPluginName(String pluginName) { + if (pluginName == null || pluginName.isEmpty()) { + throw new IllegalArgumentException("pluginName cannot be null or empty"); } - return INSTANCES.computeIfAbsent(pluginType, PluginMarker::new); - } - - /** - * Gets the static type identifier for all PluginMarker objects. - * - * @return the static type identifier - */ - public String getTypeId() { - return TYPE_ID; + return INSTANCES.computeIfAbsent(pluginName, PluginMarker::new); } /** - * Gets the plugin type this marker is intended for. + * Gets the plugin name this marker is intended for. * This should match the ObjectType.name() of the target plugin. * - * @return the plugin type identifier + * @return the plugin name identifier */ - public String getPluginType() { - return pluginType; + public String getPluginName() { + return pluginName; } @Override public String toString() { - return "PluginMarker{typeId='" + TYPE_ID + "', pluginType='" + pluginType + "'}"; + return "PluginMarker{pluginName='" + pluginName + "'}"; } } diff --git a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto index 12d9aff1831..177ff1bde41 100644 --- a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto +++ b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto @@ -77,6 +77,6 @@ message SetExecutionContextResponse { message RemoteFileSourcePluginFetchRequest { io.deephaven.proto.backplane.grpc.Ticket result_id = 1; - // The plugin type identifier to create the PluginMarker for. - string plugin_type = 2; + // The plugin name to create the PluginMarker for. + string plugin_name = 2; } \ No newline at end of file diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index baf9b21c3cb..2896583e86e 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -66,7 +66,7 @@ private JsRemoteFileSourceService(JsWidget widget) { */ @JsIgnore public static Promise fetchPlugin(@TsTypeRef(Object.class) WorkerConnection connection) { - String pluginType = "DeephavenGroovyRemoteFileSourcePlugin"; + String pluginName = "DeephavenPythonRemoteFileSourcePlugin"; // Create a new export ticket for the result Ticket resultTicket = connection.getTickets().newExportTicket(); @@ -74,7 +74,7 @@ public static Promise fetchPlugin(@TsTypeRef(Object.c // Create the fetch request RemoteFileSourcePluginFetchRequest fetchRequest = new RemoteFileSourcePluginFetchRequest(); fetchRequest.setResultId(resultTicket); - fetchRequest.setPluginType(pluginType); + fetchRequest.setPluginName(pluginName); // Serialize the request to bytes Uint8Array innerRequestBytes = fetchRequest.serializeBinary(); @@ -110,7 +110,7 @@ public static Promise fetchPlugin(@TsTypeRef(Object.c // The type must match RemoteFileSourcePlugin.name() TypedTicket typedTicket = new TypedTicket(); typedTicket.setTicket(dhTicket); - typedTicket.setType(pluginType); + typedTicket.setType(pluginName); JsWidget widget = new JsWidget(connection, typedTicket); diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java index 224a65446f8..bd48a2068e5 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java @@ -86,14 +86,14 @@ static RemoteFileSourcePluginFetchRequest.ToObjectReturnType create() { RemoteFileSourcePluginFetchRequest.ToObjectReturnType.ResultIdFieldType getResultId(); @JsProperty - String getPluginType(); + String getPluginName(); @JsProperty void setResultId( RemoteFileSourcePluginFetchRequest.ToObjectReturnType.ResultIdFieldType resultId); @JsProperty - void setPluginType(String pluginType); + void setPluginName(String pluginName); } @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) @@ -165,14 +165,14 @@ static RemoteFileSourcePluginFetchRequest.ToObjectReturnType0 create() { RemoteFileSourcePluginFetchRequest.ToObjectReturnType0.ResultIdFieldType getResultId(); @JsProperty - String getPluginType(); + String getPluginName(); @JsProperty void setResultId( RemoteFileSourcePluginFetchRequest.ToObjectReturnType0.ResultIdFieldType resultId); @JsProperty - void setPluginType(String pluginType); + void setPluginName(String pluginName); } public static native RemoteFileSourcePluginFetchRequest deserializeBinary(Uint8Array bytes); @@ -190,7 +190,7 @@ public static native RemoteFileSourcePluginFetchRequest.ToObjectReturnType toObj public native Ticket getResultId(); - public native String getPluginType(); + public native String getPluginName(); public native boolean hasResultId(); @@ -200,8 +200,7 @@ public static native RemoteFileSourcePluginFetchRequest.ToObjectReturnType toObj public native void setResultId(Ticket value); - - public native void setPluginType(String value); + public native void setPluginName(String value); public native RemoteFileSourcePluginFetchRequest.ToObjectReturnType0 toObject(); From d94696380e271ce8110c4ebcd84708e9feb36a7f Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Fri, 12 Dec 2025 14:17:28 -0600 Subject: [PATCH 24/57] Fixed incorrect plugin name (#DH-20578) --- .../client/api/remotefilesource/JsRemoteFileSourceService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index 2896583e86e..e825a842ef9 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -66,7 +66,7 @@ private JsRemoteFileSourceService(JsWidget widget) { */ @JsIgnore public static Promise fetchPlugin(@TsTypeRef(Object.class) WorkerConnection connection) { - String pluginName = "DeephavenPythonRemoteFileSourcePlugin"; + String pluginName = "DeephavenGroovyRemoteFileSourcePlugin"; // Create a new export ticket for the result Ticket resultTicket = connection.getTickets().newExportTicket(); From 183d21cf0f5d620d8e94ce6c637698ba926b77ff Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Tue, 16 Dec 2025 11:22:42 -0600 Subject: [PATCH 25/57] Cleanup (#DH-20578) --- .../deephaven/plugin/type/PluginMarker.java | 2 -- .../JsRemoteFileSourceService.java | 26 +++++++++++++------ 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java b/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java index be32f2227fb..570af422ecc 100644 --- a/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java +++ b/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java @@ -5,12 +5,10 @@ /** * A generic marker object for plugin exports that can be shared across multiple plugin types. - * * IMPORTANT: The pluginName field is required because ObjectTypeLookup.findObjectType() * returns the FIRST plugin where isType() returns true. Without plugin-specific identification * in isType(), multiple plugins using PluginMarker would conflict, and whichever is registered * first would intercept all PluginMarker instances. - * * This class uses a singleton pattern - one instance per pluginName. */ public class PluginMarker { diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index e825a842ef9..8244d6da205 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -59,15 +59,14 @@ private JsRemoteFileSourceService(JsWidget widget) { } /** - * Fetches a RemoteFileSource plugin instance from the server and establishes a message stream connection. + * Fetches the FlightInfo for the plugin fetch command. * - * @param connection the worker connection to use for communication - * @return a promise that resolves to a RemoteFileSourceService instance with an active message stream + * @param connection the worker connection to use + * @param pluginName the name of the plugin to fetch + * @return a promise that resolves to the FlightInfo for the plugin fetch */ @JsIgnore - public static Promise fetchPlugin(@TsTypeRef(Object.class) WorkerConnection connection) { - String pluginName = "DeephavenGroovyRemoteFileSourcePlugin"; - + private static Promise fetchPluginFlightInfo(WorkerConnection connection, String pluginName) { // Create a new export ticket for the result Ticket resultTicket = connection.getTickets().newExportTicket(); @@ -91,7 +90,19 @@ public static Promise fetchPlugin(@TsTypeRef(Object.c // Send the getFlightInfo request return Callbacks.grpcUnaryPromise( - c -> connection.flightServiceClient().getFlightInfo(descriptor, connection.metadata(), c::apply)) + c -> connection.flightServiceClient().getFlightInfo(descriptor, connection.metadata(), c::apply)); + } + + /** + * Fetches a RemoteFileSource plugin instance from the server and establishes a message stream connection. + * + * @param connection the worker connection to use for communication + * @return a promise that resolves to a RemoteFileSourceService instance with an active message stream + */ + @JsIgnore + public static Promise fetchPlugin(@TsTypeRef(Object.class) WorkerConnection connection) { + String pluginName = "DeephavenGroovyRemoteFileSourcePlugin"; + return fetchPluginFlightInfo(connection, pluginName) .then(flightInfo -> { // The first endpoint contains the ticket for the plugin instance. // This is the standard Flight pattern: we passed resultTicket in the request, @@ -365,4 +376,3 @@ private static int writeVarint(Uint8Array buffer, int pos, int value) { return pos; } } - From 10267c28348de89b164d7c65af2850e5224415c7 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 17 Dec 2025 15:45:54 -0600 Subject: [PATCH 26/57] Renamed event (#DH-20578) --- .../io/deephaven/engine/util/RemoteFileSourceClassLoader.java | 1 + .../api/remotefilesource/JsRemoteFileSourceService.java | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java index fa8b5bf03bd..a3235ad9450 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java @@ -55,6 +55,7 @@ protected URL findResource(String name) { Boolean canSource = provider.canSourceResource(name) .orTimeout(5, TimeUnit.SECONDS) .get(); + if (Boolean.TRUE.equals(canSource)) { return new URL(null, "remotefile://" + name, new RemoteFileURLStreamHandler(provider, name)); } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index 8244d6da205..ba4367ecb0d 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -45,7 +45,7 @@ @JsType(namespace = "dh.remotefilesource", name = "RemoteFileSourceService") public class JsRemoteFileSourceService extends HasEventHandling { public static final String EVENT_MESSAGE = "message"; - public static final String EVENT_REQUEST = "request"; + public static final String EVENT_REQUEST_SOURCE = "requestsource"; private final JsWidget widget; @@ -152,7 +152,7 @@ private Promise connect() { // If server has requested a resource from the client, fire request event RemoteFileSourceMetaRequest request = message.getMetaRequest(); - DomGlobal.setTimeout(ignore -> fireEvent(EVENT_REQUEST, + DomGlobal.setTimeout(ignore -> fireEvent(EVENT_REQUEST_SOURCE, new ResourceRequestEvent(message.getRequestId(), request)), 0); } else if (message.hasSetExecutionContextResponse()) { // Server acknowledged execution context was set From 9a29addff8a5d4bff2f2cc8443f0774ff8d4767a Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Thu, 18 Dec 2025 15:17:44 -0600 Subject: [PATCH 27/57] Passing in relative file paths instead of top-level folder (#DH-20578) --- .../RemoteFileSourceMessageStream.java | 40 +++++++++---------- .../proto/remotefilesource.proto | 6 +-- .../JsRemoteFileSourceService.java | 37 +++++++++++------ .../SetExecutionContextRequest.java | 11 +++-- 4 files changed, 52 insertions(+), 42 deletions(-) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java index ab5232ae9ee..1067966d2c0 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java @@ -67,16 +67,14 @@ public java.util.concurrent.CompletableFuture canSourceResource(String return java.util.concurrent.CompletableFuture.completedFuture(false); } - java.util.List topLevelPackages = context.getTopLevelPackages(); - if (topLevelPackages.isEmpty()) { + java.util.List resourcePaths = context.getResourcePaths(); + if (resourcePaths.isEmpty()) { return java.util.concurrent.CompletableFuture.completedFuture(false); } - String resourcePath = resourceName.replace('\\', '/'); - - for (String topLevelPackage : topLevelPackages) { - String packagePath = topLevelPackage.replace('.', '/'); - if (resourcePath.startsWith(packagePath + "/") || resourcePath.startsWith(packagePath)) { + // Resource names from ClassLoader always use forward slashes, not backslashes + for (String contextResourcePath : resourcePaths) { + if (resourceName.equals(contextResourcePath)) { log.info().append("✅ Can source: ").append(resourceName).endl(); return java.util.concurrent.CompletableFuture.completedFuture(true); } @@ -137,11 +135,11 @@ public boolean isActive() { // Static methods for execution context management /** - * Sets the execution context with the active message stream and top-level packages. + * Sets the execution context with the active message stream and resource paths. * This should be called when a script execution begins. * * @param messageStream the message stream to set as active (must not be null) - * @param packages list of top-level package names to resolve from remote source + * @param packages list of resource paths to resolve from remote source * @throws IllegalArgumentException if messageStream is null (use clearExecutionContext() instead) */ public static void setExecutionContext(RemoteFileSourceMessageStream messageStream, java.util.List packages) { @@ -151,7 +149,7 @@ public static void setExecutionContext(RemoteFileSourceMessageStream messageStre executionContext = new RemoteFileSourceExecutionContext(messageStream, packages); log.info().append("Set execution context with ") - .append(packages != null ? packages.size() : 0).append(" top-level packages").endl(); + .append(packages != null ? packages.size() : 0).append(" resource paths").endl(); } /** @@ -217,10 +215,10 @@ public void onData(ByteBuffer payload, Object... references) throws ObjectCommun } } else if (message.hasSetExecutionContext()) { // Client is requesting this message stream to become active - java.util.List packages = message.getSetExecutionContext().getTopLevelPackagesList(); + java.util.List packages = message.getSetExecutionContext().getResourcePathsList(); setExecutionContext(this, packages); log.info().append("Client set execution context for this message stream with ") - .append(packages.size()).append(" top-level packages").endl(); + .append(packages.size()).append(" resource paths").endl(); // Send acknowledgment back to client SetExecutionContextResponse response = SetExecutionContextResponse.newBuilder() @@ -318,24 +316,24 @@ public void testRequestResource(String resourceName) { /** * Encapsulates the execution context for remote file source operations. - * This includes the currently active message stream and the top-level packages + * This includes the currently active message stream and the resource paths * that should be resolved from the remote source. * This class is immutable - a new instance is created each time the context changes. */ public static class RemoteFileSourceExecutionContext { private final RemoteFileSourceMessageStream activeMessageStream; - private final java.util.List topLevelPackages; + private final java.util.List resourcePaths; /** * Creates a new execution context. * * @param activeMessageStream the active message stream - * @param topLevelPackages list of top-level package names to resolve from remote source + * @param resourcePaths list of resource paths to resolve from remote source */ public RemoteFileSourceExecutionContext(RemoteFileSourceMessageStream activeMessageStream, - java.util.List topLevelPackages) { + java.util.List resourcePaths) { this.activeMessageStream = activeMessageStream; - this.topLevelPackages = topLevelPackages != null ? topLevelPackages : java.util.Collections.emptyList(); + this.resourcePaths = resourcePaths != null ? resourcePaths : java.util.Collections.emptyList(); } /** @@ -348,12 +346,12 @@ public RemoteFileSourceMessageStream getActiveMessageStream() { } /** - * Gets the top-level package names that should be resolved from the remote source. + * Gets the resource paths that should be resolved from the remote source. * - * @return a copy of the list of top-level package names + * @return a copy of the list of resource paths */ - public java.util.List getTopLevelPackages() { - return new java.util.ArrayList<>(topLevelPackages); + public java.util.List getResourcePaths() { + return new java.util.ArrayList<>(resourcePaths); } } } diff --git a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto index 177ff1bde41..b383a776e6b 100644 --- a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto +++ b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto @@ -62,9 +62,9 @@ message RemoteFileSourceMetaResponse { // Request to set the execution context for script execution message SetExecutionContextRequest { - // Top-level package names that should be resolved from the remote source - // (e.g., ["com", "org"]) - repeated string top_level_packages = 1; + // Resource paths that should be resolved from the remote source + // (e.g., ["com/example/Test.groovy", "org/mycompany/Utils.groovy"]) + repeated string resource_paths = 1; } // Response acknowledging execution context was set diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index ba4367ecb0d..c91ce77fefa 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -33,6 +33,7 @@ import jsinterop.annotations.JsProperty; import jsinterop.annotations.JsType; import jsinterop.base.Js; +import org.jetbrains.annotations.NotNull; import java.util.HashMap; import java.util.Map; @@ -194,11 +195,11 @@ public void testBidirectionalCommunication(String resourceName) { * Sets the execution context on the server to identify this message stream as active * for script execution. * - * @param topLevelPackages array of top-level package names to resolve from remote source (e.g., ["com.example", "org.mycompany"]) + * @param resourcePaths array of resource paths to resolve from remote source (e.g., ["com/example/Test.groovy", "org/mycompany/Utils.groovy"]) * @return a promise that resolves to true if the server successfully set the execution context, false otherwise */ @JsMethod - public Promise setExecutionContext(@JsOptional String[] topLevelPackages) { + public Promise setExecutionContext(@JsOptional String[] resourcePaths) { return new Promise<>((resolve, reject) -> { // Generate a unique request ID String requestId = "setExecutionContext-" + (requestIdCounter++); @@ -206,19 +207,31 @@ public Promise setExecutionContext(@JsOptional String[] topLevelPackage // Store the resolve callback to call when we get the acknowledgment pendingSetExecutionContextRequests.put(requestId, resolve); - SetExecutionContextRequest setContextRequest = new SetExecutionContextRequest(); + RemoteFileSourceClientRequest clientRequest = getSetExecutionContextRequest(resourcePaths, requestId); + sendClientRequest(clientRequest); + }); + } - if (topLevelPackages != null && topLevelPackages.length > 0) { - for (String pkg : topLevelPackages) { - setContextRequest.addTopLevelPackages(pkg); - } + /** + * Helper method to build a RemoteFileSourceClientRequest for setting execution context. + * + * @param resourcePaths array of resource paths to resolve + * @param requestId unique request ID + * @return the constructed RemoteFileSourceClientRequest + */ + private static @NotNull RemoteFileSourceClientRequest getSetExecutionContextRequest(String[] resourcePaths, String requestId) { + SetExecutionContextRequest setContextRequest = new SetExecutionContextRequest(); + + if (resourcePaths != null) { + for (String resourcePath : resourcePaths) { + setContextRequest.addResourcePaths(resourcePath); } + } - RemoteFileSourceClientRequest clientRequest = new RemoteFileSourceClientRequest(); - clientRequest.setRequestId(requestId); - clientRequest.setSetExecutionContext(setContextRequest); - sendClientRequest(clientRequest); - }); + RemoteFileSourceClientRequest clientRequest = new RemoteFileSourceClientRequest(); + clientRequest.setRequestId(requestId); + clientRequest.setSetExecutionContext(setContextRequest); + return clientRequest; } /** diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java index 3748185297b..8967670152f 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java @@ -21,17 +21,16 @@ public static native SetExecutionContextRequest deserializeBinaryFromReader( public static native void serializeBinaryToWriter( SetExecutionContextRequest message, Object writer); - public native void clearTopLevelPackagesList(); + public native void clearResourcePathsList(); - public native JsArray getTopLevelPackagesList(); + public native JsArray getResourcePathsList(); public native Uint8Array serializeBinary(); + public native void setResourcePathsList(JsArray value); - public native void setTopLevelPackagesList(JsArray value); + public native String addResourcePaths(String value); - public native void addTopLevelPackages(String value); - - public native void addTopLevelPackages(String value, double index); + public native String addResourcePaths(String value, double index); } From 13b186aa9e841f5c989d1a603d3544811022af88 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Thu, 18 Dec 2025 17:20:49 -0600 Subject: [PATCH 28/57] Added remotefilesource plugin to server build.gradle (#DH-20578) --- server/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/server/build.gradle b/server/build.gradle index f59344a3cdf..564093a1f3b 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -93,6 +93,7 @@ dependencies { runtimeOnly project(':plugin-figure') runtimeOnly project(':plugin-partitionedtable') runtimeOnly project(':plugin-hierarchicaltable') + runtimeOnly project(':plugin-remotefilesource') implementation project(':plugin-gc-app') api platform(libs.grpc.bom) From b834ae54593440b0e84cf6f4493ecd91897dda94 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Fri, 19 Dec 2025 13:56:20 -0600 Subject: [PATCH 29/57] Regenerate bindings --- .../RemoteFileSourceClientRequest.java | 280 +++++++++++++++++- .../RemoteFileSourceMetaRequest.java | 39 ++- .../RemoteFileSourceMetaResponse.java | 228 +++++++++++++- .../RemoteFileSourcePluginFetchRequest.java | 24 +- .../RemoteFileSourceServerRequest.java | 134 ++++++++- .../SetExecutionContextRequest.java | 59 +++- .../SetExecutionContextResponse.java | 41 ++- .../RequestCase.java | 15 + .../RequestCase.java | 8 +- 9 files changed, 777 insertions(+), 51 deletions(-) create mode 100644 web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceclientrequest/RequestCase.java rename web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/{remotefilesourcepluginrequest => remotefilesourceserverrequest}/RequestCase.java (65%) diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java index c6426ab86d5..314b642900a 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java @@ -3,15 +3,266 @@ // package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; +import elemental2.core.JsArray; import elemental2.core.Uint8Array; +import jsinterop.annotations.JsOverlay; import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsProperty; import jsinterop.annotations.JsType; +import jsinterop.base.Js; +import jsinterop.base.JsPropertyMap; @JsType( isNative = true, name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceClientRequest", namespace = JsPackage.GLOBAL) public class RemoteFileSourceClientRequest { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface MetaResponseFieldType { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface GetContentUnionType { + @JsOverlay + static RemoteFileSourceClientRequest.ToObjectReturnType.MetaResponseFieldType.GetContentUnionType of( + Object o) { + return Js.cast(o); + } + + @JsOverlay + default String asString() { + return Js.asString(this); + } + + @JsOverlay + default Uint8Array asUint8Array() { + return Js.cast(this); + } + + @JsOverlay + default boolean isString() { + return (Object) this instanceof String; + } + + @JsOverlay + default boolean isUint8Array() { + return (Object) this instanceof Uint8Array; + } + } + + @JsOverlay + static RemoteFileSourceClientRequest.ToObjectReturnType.MetaResponseFieldType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + RemoteFileSourceClientRequest.ToObjectReturnType.MetaResponseFieldType.GetContentUnionType getContent(); + + @JsProperty + String getError(); + + @JsProperty + boolean isFound(); + + @JsProperty + void setContent( + RemoteFileSourceClientRequest.ToObjectReturnType.MetaResponseFieldType.GetContentUnionType content); + + @JsOverlay + default void setContent(String content) { + setContent( + Js.uncheckedCast( + content)); + } + + @JsOverlay + default void setContent(Uint8Array content) { + setContent( + Js.uncheckedCast( + content)); + } + + @JsProperty + void setError(String error); + + @JsProperty + void setFound(boolean found); + } + + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface SetExecutionContextFieldType { + @JsOverlay + static RemoteFileSourceClientRequest.ToObjectReturnType.SetExecutionContextFieldType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + JsArray getResourcePathsList(); + + @JsProperty + void setResourcePathsList(JsArray resourcePathsList); + + @JsOverlay + default void setResourcePathsList(String[] resourcePathsList) { + setResourcePathsList(Js.>uncheckedCast(resourcePathsList)); + } + } + + @JsOverlay + static RemoteFileSourceClientRequest.ToObjectReturnType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + RemoteFileSourceClientRequest.ToObjectReturnType.MetaResponseFieldType getMetaResponse(); + + @JsProperty + String getRequestId(); + + @JsProperty + RemoteFileSourceClientRequest.ToObjectReturnType.SetExecutionContextFieldType getSetExecutionContext(); + + @JsProperty + String getTestCommand(); + + @JsProperty + void setMetaResponse( + RemoteFileSourceClientRequest.ToObjectReturnType.MetaResponseFieldType metaResponse); + + @JsProperty + void setRequestId(String requestId); + + @JsProperty + void setSetExecutionContext( + RemoteFileSourceClientRequest.ToObjectReturnType.SetExecutionContextFieldType setExecutionContext); + + @JsProperty + void setTestCommand(String testCommand); + } + + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType0 { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface MetaResponseFieldType { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface GetContentUnionType { + @JsOverlay + static RemoteFileSourceClientRequest.ToObjectReturnType0.MetaResponseFieldType.GetContentUnionType of( + Object o) { + return Js.cast(o); + } + + @JsOverlay + default String asString() { + return Js.asString(this); + } + + @JsOverlay + default Uint8Array asUint8Array() { + return Js.cast(this); + } + + @JsOverlay + default boolean isString() { + return (Object) this instanceof String; + } + + @JsOverlay + default boolean isUint8Array() { + return (Object) this instanceof Uint8Array; + } + } + + @JsOverlay + static RemoteFileSourceClientRequest.ToObjectReturnType0.MetaResponseFieldType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + RemoteFileSourceClientRequest.ToObjectReturnType0.MetaResponseFieldType.GetContentUnionType getContent(); + + @JsProperty + String getError(); + + @JsProperty + boolean isFound(); + + @JsProperty + void setContent( + RemoteFileSourceClientRequest.ToObjectReturnType0.MetaResponseFieldType.GetContentUnionType content); + + @JsOverlay + default void setContent(String content) { + setContent( + Js.uncheckedCast( + content)); + } + + @JsOverlay + default void setContent(Uint8Array content) { + setContent( + Js.uncheckedCast( + content)); + } + + @JsProperty + void setError(String error); + + @JsProperty + void setFound(boolean found); + } + + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface SetExecutionContextFieldType { + @JsOverlay + static RemoteFileSourceClientRequest.ToObjectReturnType0.SetExecutionContextFieldType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + JsArray getResourcePathsList(); + + @JsProperty + void setResourcePathsList(JsArray resourcePathsList); + + @JsOverlay + default void setResourcePathsList(String[] resourcePathsList) { + setResourcePathsList(Js.>uncheckedCast(resourcePathsList)); + } + } + + @JsOverlay + static RemoteFileSourceClientRequest.ToObjectReturnType0 create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + RemoteFileSourceClientRequest.ToObjectReturnType0.MetaResponseFieldType getMetaResponse(); + + @JsProperty + String getRequestId(); + + @JsProperty + RemoteFileSourceClientRequest.ToObjectReturnType0.SetExecutionContextFieldType getSetExecutionContext(); + + @JsProperty + String getTestCommand(); + + @JsProperty + void setMetaResponse( + RemoteFileSourceClientRequest.ToObjectReturnType0.MetaResponseFieldType metaResponse); + + @JsProperty + void setRequestId(String requestId); + + @JsProperty + void setSetExecutionContext( + RemoteFileSourceClientRequest.ToObjectReturnType0.SetExecutionContextFieldType setExecutionContext); + + @JsProperty + void setTestCommand(String testCommand); + } + public static native RemoteFileSourceClientRequest deserializeBinary(Uint8Array bytes); public static native RemoteFileSourceClientRequest deserializeBinaryFromReader( @@ -20,41 +271,46 @@ public static native RemoteFileSourceClientRequest deserializeBinaryFromReader( public static native void serializeBinaryToWriter( RemoteFileSourceClientRequest message, Object writer); - public native void clearRequestId(); + public static native RemoteFileSourceClientRequest.ToObjectReturnType toObject( + boolean includeInstance, RemoteFileSourceClientRequest msg); public native void clearMetaResponse(); - public native void clearTestCommand(); - public native void clearSetExecutionContext(); - public native String getRequestId(); + public native void clearTestCommand(); public native RemoteFileSourceMetaResponse getMetaResponse(); - public native String getTestCommand(); + public native int getRequestCase(); + + public native String getRequestId(); public native SetExecutionContextRequest getSetExecutionContext(); - public native boolean hasMetaResponse(); + public native String getTestCommand(); - public native boolean hasTestCommand(); + public native boolean hasMetaResponse(); public native boolean hasSetExecutionContext(); - public native Uint8Array serializeBinary(); + public native boolean hasTestCommand(); - public native void setRequestId(String value); + public native Uint8Array serializeBinary(); public native void setMetaResponse(); public native void setMetaResponse(RemoteFileSourceMetaResponse value); - - public native void setTestCommand(String value); + public native void setRequestId(String value); public native void setSetExecutionContext(); public native void setSetExecutionContext(SetExecutionContextRequest value); -} + public native void setTestCommand(String value); + + public native RemoteFileSourceClientRequest.ToObjectReturnType0 toObject(); + + public native RemoteFileSourceClientRequest.ToObjectReturnType0 toObject(boolean includeInstance); +} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaRequest.java index 303d1a93cb6..eed1869744a 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaRequest.java @@ -4,14 +4,46 @@ package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; import elemental2.core.Uint8Array; +import jsinterop.annotations.JsOverlay; import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsProperty; import jsinterop.annotations.JsType; +import jsinterop.base.Js; +import jsinterop.base.JsPropertyMap; @JsType( isNative = true, name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceMetaRequest", namespace = JsPackage.GLOBAL) public class RemoteFileSourceMetaRequest { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType { + @JsOverlay + static RemoteFileSourceMetaRequest.ToObjectReturnType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + String getResourceName(); + + @JsProperty + void setResourceName(String resourceName); + } + + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType0 { + @JsOverlay + static RemoteFileSourceMetaRequest.ToObjectReturnType0 create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + String getResourceName(); + + @JsProperty + void setResourceName(String resourceName); + } + public static native RemoteFileSourceMetaRequest deserializeBinary(Uint8Array bytes); public static native RemoteFileSourceMetaRequest deserializeBinaryFromReader( @@ -20,11 +52,16 @@ public static native RemoteFileSourceMetaRequest deserializeBinaryFromReader( public static native void serializeBinaryToWriter( RemoteFileSourceMetaRequest message, Object writer); - public native void clearResourceName(); + public static native RemoteFileSourceMetaRequest.ToObjectReturnType toObject( + boolean includeInstance, RemoteFileSourceMetaRequest msg); public native String getResourceName(); public native Uint8Array serializeBinary(); public native void setResourceName(String value); + + public native RemoteFileSourceMetaRequest.ToObjectReturnType0 toObject(); + + public native RemoteFileSourceMetaRequest.ToObjectReturnType0 toObject(boolean includeInstance); } diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaResponse.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaResponse.java index 157c0fad3fb..d06ac259b4a 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaResponse.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaResponse.java @@ -4,14 +4,210 @@ package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; import elemental2.core.Uint8Array; +import jsinterop.annotations.JsOverlay; import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsProperty; import jsinterop.annotations.JsType; +import jsinterop.base.Js; +import jsinterop.base.JsPropertyMap; @JsType( isNative = true, name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceMetaResponse", namespace = JsPackage.GLOBAL) public class RemoteFileSourceMetaResponse { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface GetContentUnionType { + @JsOverlay + static RemoteFileSourceMetaResponse.GetContentUnionType of(Object o) { + return Js.cast(o); + } + + @JsOverlay + default String asString() { + return Js.asString(this); + } + + @JsOverlay + default Uint8Array asUint8Array() { + return Js.cast(this); + } + + @JsOverlay + default boolean isString() { + return (Object) this instanceof String; + } + + @JsOverlay + default boolean isUint8Array() { + return (Object) this instanceof Uint8Array; + } + } + + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface SetContentValueUnionType { + @JsOverlay + static RemoteFileSourceMetaResponse.SetContentValueUnionType of(Object o) { + return Js.cast(o); + } + + @JsOverlay + default String asString() { + return Js.asString(this); + } + + @JsOverlay + default Uint8Array asUint8Array() { + return Js.cast(this); + } + + @JsOverlay + default boolean isString() { + return (Object) this instanceof String; + } + + @JsOverlay + default boolean isUint8Array() { + return (Object) this instanceof Uint8Array; + } + } + + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface GetContentUnionType { + @JsOverlay + static RemoteFileSourceMetaResponse.ToObjectReturnType.GetContentUnionType of(Object o) { + return Js.cast(o); + } + + @JsOverlay + default String asString() { + return Js.asString(this); + } + + @JsOverlay + default Uint8Array asUint8Array() { + return Js.cast(this); + } + + @JsOverlay + default boolean isString() { + return (Object) this instanceof String; + } + + @JsOverlay + default boolean isUint8Array() { + return (Object) this instanceof Uint8Array; + } + } + + @JsOverlay + static RemoteFileSourceMetaResponse.ToObjectReturnType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + RemoteFileSourceMetaResponse.ToObjectReturnType.GetContentUnionType getContent(); + + @JsProperty + String getError(); + + @JsProperty + boolean isFound(); + + @JsProperty + void setContent(RemoteFileSourceMetaResponse.ToObjectReturnType.GetContentUnionType content); + + @JsOverlay + default void setContent(String content) { + setContent( + Js.uncheckedCast( + content)); + } + + @JsOverlay + default void setContent(Uint8Array content) { + setContent( + Js.uncheckedCast( + content)); + } + + @JsProperty + void setError(String error); + + @JsProperty + void setFound(boolean found); + } + + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType0 { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface GetContentUnionType { + @JsOverlay + static RemoteFileSourceMetaResponse.ToObjectReturnType0.GetContentUnionType of(Object o) { + return Js.cast(o); + } + + @JsOverlay + default String asString() { + return Js.asString(this); + } + + @JsOverlay + default Uint8Array asUint8Array() { + return Js.cast(this); + } + + @JsOverlay + default boolean isString() { + return (Object) this instanceof String; + } + + @JsOverlay + default boolean isUint8Array() { + return (Object) this instanceof Uint8Array; + } + } + + @JsOverlay + static RemoteFileSourceMetaResponse.ToObjectReturnType0 create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + RemoteFileSourceMetaResponse.ToObjectReturnType0.GetContentUnionType getContent(); + + @JsProperty + String getError(); + + @JsProperty + boolean isFound(); + + @JsProperty + void setContent(RemoteFileSourceMetaResponse.ToObjectReturnType0.GetContentUnionType content); + + @JsOverlay + default void setContent(String content) { + setContent( + Js.uncheckedCast( + content)); + } + + @JsOverlay + default void setContent(Uint8Array content) { + setContent( + Js.uncheckedCast( + content)); + } + + @JsProperty + void setError(String error); + + @JsProperty + void setFound(boolean found); + } + public static native RemoteFileSourceMetaResponse deserializeBinary(Uint8Array bytes); public static native RemoteFileSourceMetaResponse deserializeBinaryFromReader( @@ -20,24 +216,38 @@ public static native RemoteFileSourceMetaResponse deserializeBinaryFromReader( public static native void serializeBinaryToWriter( RemoteFileSourceMetaResponse message, Object writer); - public native void clearContent(); - - public native void clearFound(); + public static native RemoteFileSourceMetaResponse.ToObjectReturnType toObject( + boolean includeInstance, RemoteFileSourceMetaResponse msg); - public native void clearError(); + public native RemoteFileSourceMetaResponse.GetContentUnionType getContent(); - public native Uint8Array getContent(); + public native String getContent_asB64(); - public native boolean getFound(); + public native Uint8Array getContent_asU8(); public native String getError(); + public native boolean getFound(); + public native Uint8Array serializeBinary(); - public native void setContent(Uint8Array value); + public native void setContent(RemoteFileSourceMetaResponse.SetContentValueUnionType value); - public native void setFound(boolean value); + @JsOverlay + public final void setContent(String value) { + setContent(Js.uncheckedCast(value)); + } + + @JsOverlay + public final void setContent(Uint8Array value) { + setContent(Js.uncheckedCast(value)); + } public native void setError(String value); -} + public native void setFound(boolean value); + + public native RemoteFileSourceMetaResponse.ToObjectReturnType0 toObject(); + + public native RemoteFileSourceMetaResponse.ToObjectReturnType0 toObject(boolean includeInstance); +} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java index bd48a2068e5..b87d9dcb742 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java @@ -82,18 +82,18 @@ static RemoteFileSourcePluginFetchRequest.ToObjectReturnType create() { return Js.uncheckedCast(JsPropertyMap.of()); } + @JsProperty + String getPluginName(); + @JsProperty RemoteFileSourcePluginFetchRequest.ToObjectReturnType.ResultIdFieldType getResultId(); @JsProperty - String getPluginName(); + void setPluginName(String pluginName); @JsProperty void setResultId( RemoteFileSourcePluginFetchRequest.ToObjectReturnType.ResultIdFieldType resultId); - - @JsProperty - void setPluginName(String pluginName); } @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) @@ -161,18 +161,18 @@ static RemoteFileSourcePluginFetchRequest.ToObjectReturnType0 create() { return Js.uncheckedCast(JsPropertyMap.of()); } + @JsProperty + String getPluginName(); + @JsProperty RemoteFileSourcePluginFetchRequest.ToObjectReturnType0.ResultIdFieldType getResultId(); @JsProperty - String getPluginName(); + void setPluginName(String pluginName); @JsProperty void setResultId( RemoteFileSourcePluginFetchRequest.ToObjectReturnType0.ResultIdFieldType resultId); - - @JsProperty - void setPluginName(String pluginName); } public static native RemoteFileSourcePluginFetchRequest deserializeBinary(Uint8Array bytes); @@ -188,20 +188,20 @@ public static native RemoteFileSourcePluginFetchRequest.ToObjectReturnType toObj public native void clearResultId(); - public native Ticket getResultId(); - public native String getPluginName(); + public native Ticket getResultId(); + public native boolean hasResultId(); public native Uint8Array serializeBinary(); + public native void setPluginName(String value); + public native void setResultId(); public native void setResultId(Ticket value); - public native void setPluginName(String value); - public native RemoteFileSourcePluginFetchRequest.ToObjectReturnType0 toObject(); public native RemoteFileSourcePluginFetchRequest.ToObjectReturnType0 toObject( diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java index 4ce84895ce4..f659e2336b2 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java @@ -4,14 +4,130 @@ package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; import elemental2.core.Uint8Array; +import jsinterop.annotations.JsOverlay; import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsProperty; import jsinterop.annotations.JsType; +import jsinterop.base.Js; +import jsinterop.base.JsPropertyMap; @JsType( isNative = true, name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceServerRequest", namespace = JsPackage.GLOBAL) public class RemoteFileSourceServerRequest { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface MetaRequestFieldType { + @JsOverlay + static RemoteFileSourceServerRequest.ToObjectReturnType.MetaRequestFieldType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + String getResourceName(); + + @JsProperty + void setResourceName(String resourceName); + } + + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface SetExecutionContextResponseFieldType { + @JsOverlay + static RemoteFileSourceServerRequest.ToObjectReturnType.SetExecutionContextResponseFieldType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + boolean isSuccess(); + + @JsProperty + void setSuccess(boolean success); + } + + @JsOverlay + static RemoteFileSourceServerRequest.ToObjectReturnType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + RemoteFileSourceServerRequest.ToObjectReturnType.MetaRequestFieldType getMetaRequest(); + + @JsProperty + String getRequestId(); + + @JsProperty + RemoteFileSourceServerRequest.ToObjectReturnType.SetExecutionContextResponseFieldType getSetExecutionContextResponse(); + + @JsProperty + void setMetaRequest( + RemoteFileSourceServerRequest.ToObjectReturnType.MetaRequestFieldType metaRequest); + + @JsProperty + void setRequestId(String requestId); + + @JsProperty + void setSetExecutionContextResponse( + RemoteFileSourceServerRequest.ToObjectReturnType.SetExecutionContextResponseFieldType setExecutionContextResponse); + } + + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType0 { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface MetaRequestFieldType { + @JsOverlay + static RemoteFileSourceServerRequest.ToObjectReturnType0.MetaRequestFieldType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + String getResourceName(); + + @JsProperty + void setResourceName(String resourceName); + } + + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface SetExecutionContextResponseFieldType { + @JsOverlay + static RemoteFileSourceServerRequest.ToObjectReturnType0.SetExecutionContextResponseFieldType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + boolean isSuccess(); + + @JsProperty + void setSuccess(boolean success); + } + + @JsOverlay + static RemoteFileSourceServerRequest.ToObjectReturnType0 create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + RemoteFileSourceServerRequest.ToObjectReturnType0.MetaRequestFieldType getMetaRequest(); + + @JsProperty + String getRequestId(); + + @JsProperty + RemoteFileSourceServerRequest.ToObjectReturnType0.SetExecutionContextResponseFieldType getSetExecutionContextResponse(); + + @JsProperty + void setMetaRequest( + RemoteFileSourceServerRequest.ToObjectReturnType0.MetaRequestFieldType metaRequest); + + @JsProperty + void setRequestId(String requestId); + + @JsProperty + void setSetExecutionContextResponse( + RemoteFileSourceServerRequest.ToObjectReturnType0.SetExecutionContextResponseFieldType setExecutionContextResponse); + } + public static native RemoteFileSourceServerRequest deserializeBinary(Uint8Array bytes); public static native RemoteFileSourceServerRequest deserializeBinaryFromReader( @@ -20,16 +136,19 @@ public static native RemoteFileSourceServerRequest deserializeBinaryFromReader( public static native void serializeBinaryToWriter( RemoteFileSourceServerRequest message, Object writer); - public native void clearRequestId(); + public static native RemoteFileSourceServerRequest.ToObjectReturnType toObject( + boolean includeInstance, RemoteFileSourceServerRequest msg); public native void clearMetaRequest(); public native void clearSetExecutionContextResponse(); - public native String getRequestId(); - public native RemoteFileSourceMetaRequest getMetaRequest(); + public native int getRequestCase(); + + public native String getRequestId(); + public native SetExecutionContextResponse getSetExecutionContextResponse(); public native boolean hasMetaRequest(); @@ -38,14 +157,17 @@ public static native void serializeBinaryToWriter( public native Uint8Array serializeBinary(); - public native void setRequestId(String value); - public native void setMetaRequest(); public native void setMetaRequest(RemoteFileSourceMetaRequest value); + public native void setRequestId(String value); + public native void setSetExecutionContextResponse(); public native void setSetExecutionContextResponse(SetExecutionContextResponse value); -} + public native RemoteFileSourceServerRequest.ToObjectReturnType0 toObject(); + + public native RemoteFileSourceServerRequest.ToObjectReturnType0 toObject(boolean includeInstance); +} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java index 8967670152f..e8c069eb1ee 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java @@ -5,14 +5,56 @@ import elemental2.core.JsArray; import elemental2.core.Uint8Array; +import jsinterop.annotations.JsOverlay; import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsProperty; import jsinterop.annotations.JsType; +import jsinterop.base.Js; +import jsinterop.base.JsPropertyMap; @JsType( isNative = true, name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.SetExecutionContextRequest", namespace = JsPackage.GLOBAL) public class SetExecutionContextRequest { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType { + @JsOverlay + static SetExecutionContextRequest.ToObjectReturnType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + JsArray getResourcePathsList(); + + @JsProperty + void setResourcePathsList(JsArray resourcePathsList); + + @JsOverlay + default void setResourcePathsList(String[] resourcePathsList) { + setResourcePathsList(Js.>uncheckedCast(resourcePathsList)); + } + } + + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType0 { + @JsOverlay + static SetExecutionContextRequest.ToObjectReturnType0 create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + JsArray getResourcePathsList(); + + @JsProperty + void setResourcePathsList(JsArray resourcePathsList); + + @JsOverlay + default void setResourcePathsList(String[] resourcePathsList) { + setResourcePathsList(Js.>uncheckedCast(resourcePathsList)); + } + } + public static native SetExecutionContextRequest deserializeBinary(Uint8Array bytes); public static native SetExecutionContextRequest deserializeBinaryFromReader( @@ -21,6 +63,13 @@ public static native SetExecutionContextRequest deserializeBinaryFromReader( public static native void serializeBinaryToWriter( SetExecutionContextRequest message, Object writer); + public static native SetExecutionContextRequest.ToObjectReturnType toObject( + boolean includeInstance, SetExecutionContextRequest msg); + + public native String addResourcePaths(String value, double index); + + public native String addResourcePaths(String value); + public native void clearResourcePathsList(); public native JsArray getResourcePathsList(); @@ -29,8 +78,12 @@ public static native void serializeBinaryToWriter( public native void setResourcePathsList(JsArray value); - public native String addResourcePaths(String value); + @JsOverlay + public final void setResourcePathsList(String[] value) { + setResourcePathsList(Js.>uncheckedCast(value)); + } - public native String addResourcePaths(String value, double index); -} + public native SetExecutionContextRequest.ToObjectReturnType0 toObject(); + public native SetExecutionContextRequest.ToObjectReturnType0 toObject(boolean includeInstance); +} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextResponse.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextResponse.java index 8c27b48e69a..09d9d1861d8 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextResponse.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextResponse.java @@ -4,14 +4,46 @@ package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; import elemental2.core.Uint8Array; +import jsinterop.annotations.JsOverlay; import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsProperty; import jsinterop.annotations.JsType; +import jsinterop.base.Js; +import jsinterop.base.JsPropertyMap; @JsType( isNative = true, name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.SetExecutionContextResponse", namespace = JsPackage.GLOBAL) public class SetExecutionContextResponse { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType { + @JsOverlay + static SetExecutionContextResponse.ToObjectReturnType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + boolean isSuccess(); + + @JsProperty + void setSuccess(boolean success); + } + + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType0 { + @JsOverlay + static SetExecutionContextResponse.ToObjectReturnType0 create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + boolean isSuccess(); + + @JsProperty + void setSuccess(boolean success); + } + public static native SetExecutionContextResponse deserializeBinary(Uint8Array bytes); public static native SetExecutionContextResponse deserializeBinaryFromReader( @@ -20,13 +52,16 @@ public static native SetExecutionContextResponse deserializeBinaryFromReader( public static native void serializeBinaryToWriter( SetExecutionContextResponse message, Object writer); - public native void clearSuccess(); + public static native SetExecutionContextResponse.ToObjectReturnType toObject( + boolean includeInstance, SetExecutionContextResponse msg); public native boolean getSuccess(); public native Uint8Array serializeBinary(); - public native void setSuccess(boolean value); -} + public native SetExecutionContextResponse.ToObjectReturnType0 toObject(); + + public native SetExecutionContextResponse.ToObjectReturnType0 toObject(boolean includeInstance); +} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceclientrequest/RequestCase.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceclientrequest/RequestCase.java new file mode 100644 index 00000000000..46db67bea6b --- /dev/null +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceclientrequest/RequestCase.java @@ -0,0 +1,15 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.remotefilesourceclientrequest; + +import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsType; + +@JsType( + isNative = true, + name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceClientRequest.RequestCase", + namespace = JsPackage.GLOBAL) +public class RequestCase { + public static int META_RESPONSE, REQUEST_NOT_SET, SET_EXECUTION_CONTEXT, TEST_COMMAND; +} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourcepluginrequest/RequestCase.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceserverrequest/RequestCase.java similarity index 65% rename from web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourcepluginrequest/RequestCase.java rename to web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceserverrequest/RequestCase.java index c263a6194cb..72cad9f6ff9 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourcepluginrequest/RequestCase.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceserverrequest/RequestCase.java @@ -1,17 +1,15 @@ // // Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending // -package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.remotefilesourcepluginrequest; +package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.remotefilesourceserverrequest; import jsinterop.annotations.JsPackage; import jsinterop.annotations.JsType; @JsType( isNative = true, - name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourcePluginRequest.RequestCase", + name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceServerRequest.RequestCase", namespace = JsPackage.GLOBAL) public class RequestCase { - public static int META, - REQUEST_NOT_SET, - SET_CONNECTION_ID; + public static int META_REQUEST, REQUEST_NOT_SET, SET_EXECUTION_CONTEXT_RESPONSE; } From 5e29bdd8bfd8c2bbfc8e506d0d55afcdbc1d7e5a Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Fri, 19 Dec 2025 14:00:57 -0600 Subject: [PATCH 30/57] FIxing compile error --- .../api/remotefilesource/JsRemoteFileSourceService.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index c91ce77fefa..845abafb560 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -283,7 +283,7 @@ public String getResourceName() { /** * Responds to this resource request with the given content. * - * @param content the resource content (string, ArrayBuffer, or typed array), or null if not found + * @param content the resource content (string or Uint8Array), or null if not found */ @JsMethod public void respond(@JsNullable Object content) { @@ -299,11 +299,11 @@ public void respond(@JsNullable Object content) { // Convert content to bytes if (content instanceof String) { - response.setContent(stringToUtf8((String) content)); + response.setContent((String) content); } else if (content instanceof Uint8Array) { response.setContent((Uint8Array) content); } else { - response.setContent(Js.uncheckedCast(content)); + throw new IllegalArgumentException("Content must be a String, Uint8Array, or null"); } } From 19b0d7a375d61b5a35a423b43889f879045f3612 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Tue, 23 Dec 2025 17:06:26 -0600 Subject: [PATCH 31/57] Replaced util with TextEncoder.encode (#DH-20578) --- .../JsRemoteFileSourceService.java | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index 845abafb560..c1c953e876d 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -8,6 +8,7 @@ import com.vertispan.tsdefs.annotations.TsTypeRef; import elemental2.core.Uint8Array; import elemental2.dom.DomGlobal; +import elemental2.dom.TextEncoder; import elemental2.promise.Promise; import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.FlightDescriptor; import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.FlightInfo; @@ -299,7 +300,8 @@ public void respond(@JsNullable Object content) { // Convert content to bytes if (content instanceof String) { - response.setContent((String) content); + TextEncoder textEncoder = new TextEncoder(); + response.setContent(textEncoder.encode((String) content)); } else if (content instanceof Uint8Array) { response.setContent((Uint8Array) content); } else { @@ -325,7 +327,8 @@ public void respond(@JsNullable Object content) { */ private static Uint8Array wrapInAny(String typeUrl, Uint8Array messageBytes) { // Encode the type_url string to UTF-8 bytes - Uint8Array typeUrlBytes = stringToUtf8(typeUrl); + TextEncoder textEncoder = new TextEncoder(); + Uint8Array typeUrlBytes = textEncoder.encode(typeUrl); // Calculate sizes for protobuf encoding // Field 1 (type_url): tag + length + data @@ -357,15 +360,6 @@ private static Uint8Array wrapInAny(String typeUrl, Uint8Array messageBytes) { return result; } - private static Uint8Array stringToUtf8(String str) { - // Simple UTF-8 encoding for ASCII-compatible strings - Uint8Array bytes = new Uint8Array(str.length()); - for (int i = 0; i < str.length(); i++) { - bytes.setAt(i, (double) str.charAt(i)); - } - return bytes; - } - private static int sizeOfVarint(int value) { if (value < 0) return 10; From 51e4f9d7efe16de3d1ee8943141b712152968a1e Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 24 Dec 2025 09:48:39 -0600 Subject: [PATCH 32/57] Cleanup (#DH-20578) --- .../JsRemoteFileSourceService.java | 162 ++++++++++++++---- 1 file changed, 129 insertions(+), 33 deletions(-) diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index c1c953e876d..b2dd397a274 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -318,67 +318,163 @@ public void respond(@JsNullable Object content) { } } + /** + * Calculates the total size needed for a protobuf length-delimited field. + *

+ * A length-delimited field consists of: + *

    + *
  • Tag (field number + wire type) encoded as a varint
    • + *
  • Length of the data encoded as a varint
    • + *
  • The actual data bytes
    • + *
+ * + * @param tag the protobuf field tag (field number << 3 | wire type) + * @param dataLength the length of the data in bytes + * @return the total number of bytes needed for this field + */ + private static int calculateFieldSize(int tag, int dataLength) { + return sizeOfVarint(tag) + sizeOfVarint(dataLength) + dataLength; + } + + /** + * Calculates how many bytes a varint encoding will require for the given value. + *

+ * Protobuf uses varint encoding where each byte stores 7 bits of data (the 8th bit is + * a continuation flag). This means: + *

    + *
  • 1 byte: 0 to 127 (2^7 - 1)
    • + *
  • 2 bytes: 128 to 16,383 (2^14 - 1)
    • + *
  • 3 bytes: 16,384 to 2,097,151 (2^21 - 1)
    • + *
  • 4 bytes: 2,097,152 to 268,435,455 (2^28 - 1)
    • + *
  • 5 bytes: 268,435,456 to 4,294,967,295 (2^35 - 1, max unsigned 32-bit)
    • + *
  • 10 bytes: negative numbers (due to sign extension)
    • + *
+ * + * @param value the integer value to encode + * @return the number of bytes required to encode the value as a varint + */ + private static int sizeOfVarint(int value) { + if (value < 0) + return 10; // Negative numbers use sign extension, always 10 bytes + if (value < 128) // 2^7 + return 1; + if (value < 16384) // 2^14 + return 2; + if (value < 2097152) // 2^21 + return 3; + if (value < 268435456) // 2^28 + return 4; + return 5; // 2^35 (max for positive 32-bit int) + } + /** * Wraps a protobuf message in a google.protobuf.Any message. + *

+ * The google.protobuf.Any message has two fields: + *

    + *
  • Field 1: type_url (string) - identifies the type of message contained
    • + *
  • Field 2: value (bytes) - the actual serialized message
    • + *
+ *

+ * This method manually encodes the Any message in protobuf binary format since the client-side + * JavaScript protobuf library doesn't provide Any.pack() like the server-side Java library does. * * @param typeUrl the type URL for the message (e.g., "type.googleapis.com/package.MessageName") * @param messageBytes the serialized protobuf message bytes * @return the serialized Any message containing the wrapped message */ private static Uint8Array wrapInAny(String typeUrl, Uint8Array messageBytes) { + // Protobuf tag constants for google.protobuf.Any message fields + // Tag format: (field_number << 3) | wire_type + // wire_type=2 means length-delimited (for strings/bytes) + final int TYPE_URL_TAG = 10; // (1 << 3) | 2 = field 1, wire type 2 + final int VALUE_TAG = 18; // (2 << 3) | 2 = field 2, wire type 2 + // Encode the type_url string to UTF-8 bytes TextEncoder textEncoder = new TextEncoder(); Uint8Array typeUrlBytes = textEncoder.encode(typeUrl); - // Calculate sizes for protobuf encoding - // Field 1 (type_url): tag + length + data - int typeUrlTag = (1 << 3) | 2; // field 1, wire type 2 (length-delimited) - int typeUrlFieldSize = sizeOfVarint(typeUrlTag) + sizeOfVarint(typeUrlBytes.length) + typeUrlBytes.length; - - // Field 2 (value): tag + length + data - int valueTag = (2 << 3) | 2; // field 2, wire type 2 (length-delimited) - int valueFieldSize = sizeOfVarint(valueTag) + sizeOfVarint(messageBytes.length) + messageBytes.length; + // Calculate sizes for protobuf binary encoding + int typeUrlFieldSize = calculateFieldSize(TYPE_URL_TAG, typeUrlBytes.length); + int valueFieldSize = calculateFieldSize(VALUE_TAG, messageBytes.length); + // Allocate buffer for the complete Any message int totalSize = typeUrlFieldSize + valueFieldSize; Uint8Array result = new Uint8Array(totalSize); int pos = 0; - // Write type_url field - pos = writeVarint(result, pos, typeUrlTag); - pos = writeVarint(result, pos, typeUrlBytes.length); - for (int i = 0; i < typeUrlBytes.length; i++) { - result.setAt(pos++, typeUrlBytes.getAt(i)); - } + // Write field 1 (type_url) in protobuf binary format + pos = writeField(result, pos, TYPE_URL_TAG, typeUrlBytes); - // Write value field - pos = writeVarint(result, pos, valueTag); - pos = writeVarint(result, pos, messageBytes.length); - for (int i = 0; i < messageBytes.length; i++) { - result.setAt(pos++, messageBytes.getAt(i)); - } + // Write field 2 (value) in protobuf binary format + writeField(result, pos, VALUE_TAG, messageBytes); return result; } - private static int sizeOfVarint(int value) { - if (value < 0) - return 10; - if (value < 128) - return 1; - if (value < 16384) - return 2; - if (value < 2097152) - return 3; - if (value < 268435456) - return 4; - return 5; + /** + * Writes a complete protobuf length-delimited field to the buffer. + *

+ * A length-delimited field consists of: + *

    + *
  • Tag (field number + wire type) encoded as a varint
    • + *
  • Length of the data encoded as a varint
    • + *
  • The actual data bytes
    • + *
+ * + * @param buffer the buffer to write to + * @param pos the starting position in the buffer + * @param tag the protobuf field tag + * @param data the data bytes to write + * @return the new position after writing the complete field + */ + private static int writeField(Uint8Array buffer, int pos, int tag, Uint8Array data) { + // Write tag and length + pos = writeVarint(buffer, pos, tag); + pos = writeVarint(buffer, pos, data.length); + // Write data bytes + for (int i = 0; i < data.length; i++) { + buffer.setAt(pos++, data.getAt(i)); + } + return pos; } + + /** + * Writes a value to the buffer as a protobuf varint (variable-length integer). + *

+ * Varint encoding works by: + *

    + *
  1. Taking the lowest 7 bits of the value
    2. + *
  2. Setting the 8th bit to 1 if more bytes follow (continuation flag)
    3. + *
  3. Writing the byte to the buffer
    4. + *
  4. Shifting the value right by 7 bits
    5. + *
  5. Repeating until the value is less than 128
    6. + *
  6. Writing the final byte without the continuation flag (8th bit = 0)
    7. + *
+ *

+ * Example: encoding 300 + *

    + *
  • 300 in binary: 100101100
    • + *
  • First byte: (300 & 0x7F) | 0x80 = 0b00101100 | 0b10000000 = 172 (0xAC)
    • + *
  • Shift: 300 >>> 7 = 2
    • + *
  • Second byte: 2 (no continuation flag)
    • + *
  • Result: [172, 2]
    • + *
+ * + * @param buffer the buffer to write to + * @param pos the starting position in the buffer + * @param value the value to encode + * @return the new position after writing + */ private static int writeVarint(Uint8Array buffer, int pos, int value) { while (value >= 128) { + // Extract lowest 7 bits and set continuation flag (8th bit = 1) buffer.setAt(pos++, (double) ((value & 0x7F) | 0x80)); - value >>>= 7; + // Shift right by 7 to process next chunk + value >>>= 7; // Unsigned right shift to handle large positive values } + // Write final byte (no continuation flag, 8th bit = 0) buffer.setAt(pos++, (double) value); return pos; } From 434f5c8226f8f3d4d7635bcb2c77ad694146d619 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 24 Dec 2025 10:17:41 -0600 Subject: [PATCH 33/57] Split out JsProtobufUtils (#DH-20578) --- .../web/client/api/JsProtobufUtils.java | 178 ++++++++++++++++++ .../JsRemoteFileSourceService.java | 164 +--------------- 2 files changed, 180 insertions(+), 162 deletions(-) create mode 100644 web/client-api/src/main/java/io/deephaven/web/client/api/JsProtobufUtils.java diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/JsProtobufUtils.java b/web/client-api/src/main/java/io/deephaven/web/client/api/JsProtobufUtils.java new file mode 100644 index 00000000000..7413aedd954 --- /dev/null +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/JsProtobufUtils.java @@ -0,0 +1,178 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.web.client.api; + +import elemental2.core.Uint8Array; +import elemental2.dom.TextEncoder; + +/** + * Utility methods for working with protobuf messages in the client-side JavaScript environment. + */ +public class JsProtobufUtils { + + private JsProtobufUtils() { + // Utility class, no instantiation + } + + /** + * Wraps a protobuf message in a google.protobuf.Any message. + *

+ * The google.protobuf.Any message has two fields: + *

    + *
  • Field 1: type_url (string) - identifies the type of message contained
    • + *
  • Field 2: value (bytes) - the actual serialized message
    • + *
+ *

+ * This method manually encodes the Any message in protobuf binary format since the client-side + * JavaScript protobuf library doesn't provide Any.pack() like the server-side Java library does. + * + * @param typeUrl the type URL for the message (e.g., "type.googleapis.com/package.MessageName") + * @param messageBytes the serialized protobuf message bytes + * @return the serialized Any message containing the wrapped message + */ + public static Uint8Array wrapInAny(String typeUrl, Uint8Array messageBytes) { + // Protobuf tag constants for google.protobuf.Any message fields + // Tag format: (field_number << 3) | wire_type + // wire_type=2 means length-delimited (for strings/bytes) + final int TYPE_URL_TAG = 10; // (1 << 3) | 2 = field 1, wire type 2 + final int VALUE_TAG = 18; // (2 << 3) | 2 = field 2, wire type 2 + + // Encode the type_url string to UTF-8 bytes + TextEncoder textEncoder = new TextEncoder(); + Uint8Array typeUrlBytes = textEncoder.encode(typeUrl); + + // Calculate sizes for protobuf binary encoding + int typeUrlFieldSize = calculateFieldSize(TYPE_URL_TAG, typeUrlBytes.length); + int valueFieldSize = calculateFieldSize(VALUE_TAG, messageBytes.length); + + // Allocate buffer for the complete Any message + int totalSize = typeUrlFieldSize + valueFieldSize; + Uint8Array result = new Uint8Array(totalSize); + int pos = 0; + + // Write field 1 (type_url) in protobuf binary format + pos = writeField(result, pos, TYPE_URL_TAG, typeUrlBytes); + + // Write field 2 (value) in protobuf binary format + writeField(result, pos, VALUE_TAG, messageBytes); + + return result; + } + + /** + * Calculates the total size needed for a protobuf length-delimited field. + *

+ * A length-delimited field consists of: + *

    + *
  • Tag (field number + wire type) encoded as a varint
    • + *
  • Length of the data encoded as a varint
    • + *
  • The actual data bytes
    • + *
+ * + * @param tag the protobuf field tag (field number << 3 | wire type) + * @param dataLength the length of the data in bytes + * @return the total number of bytes needed for this field + */ + private static int calculateFieldSize(int tag, int dataLength) { + return sizeOfVarint(tag) + sizeOfVarint(dataLength) + dataLength; + } + + /** + * Calculates how many bytes a varint encoding will require for the given value. + *

+ * Protobuf uses varint encoding where each byte stores 7 bits of data (the 8th bit is + * a continuation flag). This means: + *

    + *
  • 1 byte: 0 to 127 (2^7 - 1)
    • + *
  • 2 bytes: 128 to 16,383 (2^14 - 1)
    • + *
  • 3 bytes: 16,384 to 2,097,151 (2^21 - 1)
    • + *
  • 4 bytes: 2,097,152 to 268,435,455 (2^28 - 1)
    • + *
  • 5 bytes: 268,435,456 to 4,294,967,295 (2^35 - 1, max unsigned 32-bit)
    • + *
  • 10 bytes: negative numbers (due to sign extension)
    • + *
+ * + * @param value the integer value to encode + * @return the number of bytes required to encode the value as a varint + */ + private static int sizeOfVarint(int value) { + if (value < 0) + return 10; // Negative numbers use sign extension, always 10 bytes + if (value < 128) // 2^7 + return 1; + if (value < 16384) // 2^14 + return 2; + if (value < 2097152) // 2^21 + return 3; + if (value < 268435456) // 2^28 + return 4; + return 5; // 2^35 (max for positive 32-bit int) + } + + /** + * Writes a complete protobuf length-delimited field to the buffer. + *

+ * A length-delimited field consists of: + *

    + *
  • Tag (field number + wire type) encoded as a varint
    • + *
  • Length of the data encoded as a varint
    • + *
  • The actual data bytes
    • + *
+ * + * @param buffer the buffer to write to + * @param pos the starting position in the buffer + * @param tag the protobuf field tag + * @param data the data bytes to write + * @return the new position after writing the complete field + */ + private static int writeField(Uint8Array buffer, int pos, int tag, Uint8Array data) { + // Write tag and length + pos = writeVarint(buffer, pos, tag); + pos = writeVarint(buffer, pos, data.length); + // Write data bytes + for (int i = 0; i < data.length; i++) { + buffer.setAt(pos++, data.getAt(i)); + } + return pos; + } + + /** + * Writes a value to the buffer as a protobuf varint (variable-length integer). + *

+ * Varint encoding works by: + *

    + *
  1. Taking the lowest 7 bits of the value
    2. + *
  2. Setting the 8th bit to 1 if more bytes follow (continuation flag)
    3. + *
  3. Writing the byte to the buffer
    4. + *
  4. Shifting the value right by 7 bits
    5. + *
  5. Repeating until the value is less than 128
    6. + *
  6. Writing the final byte without the continuation flag (8th bit = 0)
    7. + *
+ *

+ * Example: encoding 300 + *

    + *
  • 300 in binary: 100101100
    • + *
  • First byte: (300 & 0x7F) | 0x80 = 0b00101100 | 0b10000000 = 172 (0xAC)
    • + *
  • Shift: 300 >>> 7 = 2
    • + *
  • Second byte: 2 (no continuation flag)
    • + *
  • Result: [172, 2]
    • + *
+ * + * @param buffer the buffer to write to + * @param pos the starting position in the buffer + * @param value the value to encode + * @return the new position after writing + */ + private static int writeVarint(Uint8Array buffer, int pos, int value) { + while (value >= 128) { + // Extract lowest 7 bits and set continuation flag (8th bit = 1) + buffer.setAt(pos++, (double) ((value & 0x7F) | 0x80)); + // Shift right by 7 to process next chunk + value >>>= 7; // Unsigned right shift to handle large positive values + } + // Write final byte (no continuation flag, 8th bit = 0) + buffer.setAt(pos++, (double) value); + return pos; + } +} + diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index b2dd397a274..dffeefae01d 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -22,6 +22,7 @@ import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.Ticket; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.TypedTicket; import io.deephaven.web.client.api.Callbacks; +import io.deephaven.web.client.api.JsProtobufUtils; import io.deephaven.web.client.api.event.Event; import io.deephaven.web.client.api.WorkerConnection; import io.deephaven.web.client.api.event.HasEventHandling; @@ -81,7 +82,7 @@ private static Promise fetchPluginFlightInfo(WorkerConnection connec Uint8Array innerRequestBytes = fetchRequest.serializeBinary(); // Wrap in google.protobuf.Any with the proper typeUrl - Uint8Array anyWrappedBytes = wrapInAny( + Uint8Array anyWrappedBytes = JsProtobufUtils.wrapInAny( "type.googleapis.com/io.deephaven.proto.backplane.grpc.RemoteFileSourcePluginFetchRequest", innerRequestBytes); @@ -317,165 +318,4 @@ public void respond(@JsNullable Object content) { sendClientRequest(clientRequest); } } - - /** - * Calculates the total size needed for a protobuf length-delimited field. - *

- * A length-delimited field consists of: - *

    - *
  • Tag (field number + wire type) encoded as a varint
    • - *
  • Length of the data encoded as a varint
    • - *
  • The actual data bytes
    • - *
- * - * @param tag the protobuf field tag (field number << 3 | wire type) - * @param dataLength the length of the data in bytes - * @return the total number of bytes needed for this field - */ - private static int calculateFieldSize(int tag, int dataLength) { - return sizeOfVarint(tag) + sizeOfVarint(dataLength) + dataLength; - } - - /** - * Calculates how many bytes a varint encoding will require for the given value. - *

- * Protobuf uses varint encoding where each byte stores 7 bits of data (the 8th bit is - * a continuation flag). This means: - *

    - *
  • 1 byte: 0 to 127 (2^7 - 1)
    • - *
  • 2 bytes: 128 to 16,383 (2^14 - 1)
    • - *
  • 3 bytes: 16,384 to 2,097,151 (2^21 - 1)
    • - *
  • 4 bytes: 2,097,152 to 268,435,455 (2^28 - 1)
    • - *
  • 5 bytes: 268,435,456 to 4,294,967,295 (2^35 - 1, max unsigned 32-bit)
    • - *
  • 10 bytes: negative numbers (due to sign extension)
    • - *
- * - * @param value the integer value to encode - * @return the number of bytes required to encode the value as a varint - */ - private static int sizeOfVarint(int value) { - if (value < 0) - return 10; // Negative numbers use sign extension, always 10 bytes - if (value < 128) // 2^7 - return 1; - if (value < 16384) // 2^14 - return 2; - if (value < 2097152) // 2^21 - return 3; - if (value < 268435456) // 2^28 - return 4; - return 5; // 2^35 (max for positive 32-bit int) - } - - /** - * Wraps a protobuf message in a google.protobuf.Any message. - *

- * The google.protobuf.Any message has two fields: - *

    - *
  • Field 1: type_url (string) - identifies the type of message contained
    • - *
  • Field 2: value (bytes) - the actual serialized message
    • - *
- *

- * This method manually encodes the Any message in protobuf binary format since the client-side - * JavaScript protobuf library doesn't provide Any.pack() like the server-side Java library does. - * - * @param typeUrl the type URL for the message (e.g., "type.googleapis.com/package.MessageName") - * @param messageBytes the serialized protobuf message bytes - * @return the serialized Any message containing the wrapped message - */ - private static Uint8Array wrapInAny(String typeUrl, Uint8Array messageBytes) { - // Protobuf tag constants for google.protobuf.Any message fields - // Tag format: (field_number << 3) | wire_type - // wire_type=2 means length-delimited (for strings/bytes) - final int TYPE_URL_TAG = 10; // (1 << 3) | 2 = field 1, wire type 2 - final int VALUE_TAG = 18; // (2 << 3) | 2 = field 2, wire type 2 - - // Encode the type_url string to UTF-8 bytes - TextEncoder textEncoder = new TextEncoder(); - Uint8Array typeUrlBytes = textEncoder.encode(typeUrl); - - // Calculate sizes for protobuf binary encoding - int typeUrlFieldSize = calculateFieldSize(TYPE_URL_TAG, typeUrlBytes.length); - int valueFieldSize = calculateFieldSize(VALUE_TAG, messageBytes.length); - - // Allocate buffer for the complete Any message - int totalSize = typeUrlFieldSize + valueFieldSize; - Uint8Array result = new Uint8Array(totalSize); - int pos = 0; - - // Write field 1 (type_url) in protobuf binary format - pos = writeField(result, pos, TYPE_URL_TAG, typeUrlBytes); - - // Write field 2 (value) in protobuf binary format - writeField(result, pos, VALUE_TAG, messageBytes); - - return result; - } - - /** - * Writes a complete protobuf length-delimited field to the buffer. - *

- * A length-delimited field consists of: - *

    - *
  • Tag (field number + wire type) encoded as a varint
    • - *
  • Length of the data encoded as a varint
    • - *
  • The actual data bytes
    • - *
- * - * @param buffer the buffer to write to - * @param pos the starting position in the buffer - * @param tag the protobuf field tag - * @param data the data bytes to write - * @return the new position after writing the complete field - */ - private static int writeField(Uint8Array buffer, int pos, int tag, Uint8Array data) { - // Write tag and length - pos = writeVarint(buffer, pos, tag); - pos = writeVarint(buffer, pos, data.length); - // Write data bytes - for (int i = 0; i < data.length; i++) { - buffer.setAt(pos++, data.getAt(i)); - } - return pos; - } - - - /** - * Writes a value to the buffer as a protobuf varint (variable-length integer). - *

- * Varint encoding works by: - *

    - *
  1. Taking the lowest 7 bits of the value
    2. - *
  2. Setting the 8th bit to 1 if more bytes follow (continuation flag)
    3. - *
  3. Writing the byte to the buffer
    4. - *
  4. Shifting the value right by 7 bits
    5. - *
  5. Repeating until the value is less than 128
    6. - *
  6. Writing the final byte without the continuation flag (8th bit = 0)
    7. - *
- *

- * Example: encoding 300 - *

    - *
  • 300 in binary: 100101100
    • - *
  • First byte: (300 & 0x7F) | 0x80 = 0b00101100 | 0b10000000 = 172 (0xAC)
    • - *
  • Shift: 300 >>> 7 = 2
    • - *
  • Second byte: 2 (no continuation flag)
    • - *
  • Result: [172, 2]
    • - *
- * - * @param buffer the buffer to write to - * @param pos the starting position in the buffer - * @param value the value to encode - * @return the new position after writing - */ - private static int writeVarint(Uint8Array buffer, int pos, int value) { - while (value >= 128) { - // Extract lowest 7 bits and set continuation flag (8th bit = 1) - buffer.setAt(pos++, (double) ((value & 0x7F) | 0x80)); - // Shift right by 7 to process next chunk - value >>>= 7; // Unsigned right shift to handle large positive values - } - // Write final byte (no continuation flag, 8th bit = 0) - buffer.setAt(pos++, (double) value); - return pos; - } } From cf85dbdb5dc3d34d2d0c403c8c03c45d7d485271 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 24 Dec 2025 10:49:04 -0600 Subject: [PATCH 34/57] Made canSourceResource synchronous (#DH-20578) --- .../engine/util/RemoteFileSourceClassLoader.java | 15 ++++++--------- .../engine/util/RemoteFileSourceProvider.java | 4 ++-- .../RemoteFileSourceMessageStream.java | 14 +++++++------- .../JsRemoteFileSourceService.java | 14 -------------- 4 files changed, 15 insertions(+), 32 deletions(-) diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java index a3235ad9450..efc658e5a1e 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java @@ -24,7 +24,8 @@ * */ public class RemoteFileSourceClassLoader extends ClassLoader { - private static final boolean DEBUG = Boolean.getBoolean("RemoteFileSourceClassLoader.debug"); + private static final long RESOURCE_TIMEOUT_SECONDS = 5; + private static volatile RemoteFileSourceClassLoader instance; private final CopyOnWriteArrayList providers = new CopyOnWriteArrayList<>(); @@ -52,15 +53,11 @@ protected URL findResource(String name) { continue; } try { - Boolean canSource = provider.canSourceResource(name) - .orTimeout(5, TimeUnit.SECONDS) - .get(); - - if (Boolean.TRUE.equals(canSource)) { + if (provider.canSourceResource(name)) { return new URL(null, "remotefile://" + name, new RemoteFileURLStreamHandler(provider, name)); } - } catch (Exception e) { - // Continue to next provider + } catch (java.net.MalformedURLException e) { + // Continue to next provider if URL creation fails } } return super.findResource(name); @@ -103,7 +100,7 @@ public void connect() throws IOException { if (!connected) { try { content = provider.requestResource(resourceName) - .orTimeout(5, TimeUnit.SECONDS) + .orTimeout(RESOURCE_TIMEOUT_SECONDS, TimeUnit.SECONDS) .get(); connected = true; } catch (Exception e) { diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java index 4c5408d7833..02e5eee5feb 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java @@ -15,9 +15,9 @@ public interface RemoteFileSourceProvider { * Check if this provider can source the given resource. * * @param resourceName the name of the resource to check (e.g., "com/example/MyClass.groovy") - * @return a CompletableFuture that resolves to true if this provider can handle the resource, false otherwise + * @return true if this provider can handle the resource, false otherwise */ - CompletableFuture canSourceResource(String resourceName); + boolean canSourceResource(String resourceName); /** * Request a resource from the remote source. diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java index 1067966d2c0..4161d4a937d 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java @@ -51,36 +51,36 @@ public RemoteFileSourceMessageStream(final ObjectType.MessageStream connection) // RemoteFileSourceProvider interface implementation - each instance is a provider @Override - public java.util.concurrent.CompletableFuture canSourceResource(String resourceName) { + public boolean canSourceResource(String resourceName) { // Only active if this instance is the currently active message stream if (!isActive()) { - return java.util.concurrent.CompletableFuture.completedFuture(false); + return false; } // Only handle .groovy source files, not compiled .class files if (!resourceName.endsWith(".groovy")) { - return java.util.concurrent.CompletableFuture.completedFuture(false); + return false; } RemoteFileSourceExecutionContext context = executionContext; if (context == null || context.getActiveMessageStream() != this) { - return java.util.concurrent.CompletableFuture.completedFuture(false); + return false; } java.util.List resourcePaths = context.getResourcePaths(); if (resourcePaths.isEmpty()) { - return java.util.concurrent.CompletableFuture.completedFuture(false); + return false; } // Resource names from ClassLoader always use forward slashes, not backslashes for (String contextResourcePath : resourcePaths) { if (resourceName.equals(contextResourcePath)) { log.info().append("✅ Can source: ").append(resourceName).endl(); - return java.util.concurrent.CompletableFuture.completedFuture(true); + return true; } } - return java.util.concurrent.CompletableFuture.completedFuture(false); + return false; } @Override diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index dffeefae01d..dd0d6ef0f09 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -179,20 +179,6 @@ private Promise connect() { return widget.refetch().then(w -> Promise.resolve(this)); } - /** - * Test method to verify bidirectional communication. - * Sends a test command to the server, which will request a resource back from the client. - * - * @param resourceName the resource name to use for the test (e.g., "com/example/Test.java") - */ - @JsMethod - public void testBidirectionalCommunication(String resourceName) { - RemoteFileSourceClientRequest clientRequest = new RemoteFileSourceClientRequest(); - clientRequest.setRequestId(""); // Empty request_id for test commands - clientRequest.setTestCommand("TEST:" + resourceName); - sendClientRequest(clientRequest); - } - /** * Sets the execution context on the server to identify this message stream as active * for script execution. From 8298fefca3d387137291c78cc2fc76e0adfd59ea Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 24 Dec 2025 10:54:10 -0600 Subject: [PATCH 35/57] renamed arg (#DH-20578) --- .../io/deephaven/engine/util/RemoteFileSourceClassLoader.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java index efc658e5a1e..25c82b41147 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java @@ -76,8 +76,8 @@ private static class RemoteFileURLStreamHandler extends URLStreamHandler { } @Override - protected URLConnection openConnection(URL u) { - return new RemoteFileURLConnection(u, provider, resourceName); + protected URLConnection openConnection(URL url) { + return new RemoteFileURLConnection(url, provider, resourceName); } } From 097607bb9b2fd6d9d2fb12e6d31c5ff885f1d33c Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Fri, 2 Jan 2026 10:22:05 -0600 Subject: [PATCH 36/57] Cleanup (#DH-20578) --- .../util/RemoteFileSourceClassLoader.java | 73 ++++++++++++++++++- 1 file changed, 69 insertions(+), 4 deletions(-) diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java index 25c82b41147..3add4e468cc 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java @@ -29,37 +29,67 @@ public class RemoteFileSourceClassLoader extends ClassLoader { private static volatile RemoteFileSourceClassLoader instance; private final CopyOnWriteArrayList providers = new CopyOnWriteArrayList<>(); + /** + * Constructs a new RemoteFileSourceClassLoader with the specified parent class loader. + * + * @param parent the parent class loader for delegation + */ public RemoteFileSourceClassLoader(ClassLoader parent) { super(parent); instance = this; } + /** + * Returns the singleton instance of the RemoteFileSourceClassLoader. + * + * @return the singleton instance, or null if not yet initialized + */ public static RemoteFileSourceClassLoader getInstance() { return instance; } + /** + * Registers a new provider that can source remote resources. + * + * @param provider the provider to register + */ public void registerProvider(RemoteFileSourceProvider provider) { providers.add(provider); } + /** + * Unregisters a previously registered provider. + * + * @param provider the provider to unregister + */ public void unregisterProvider(RemoteFileSourceProvider provider) { providers.remove(provider); } + /** + * Finds the resource with the specified name by checking registered providers. + * + *

This method iterates through all registered providers to see if any can source the requested resource. + * If a provider can handle the resource, a custom URL with protocol "remotefile://" is returned. + * If no provider can handle the resource, the request is delegated to the parent class loader. + * + * @param name the resource name + * @return a URL for reading the resource, or null if the resource could not be found + */ @Override protected URL findResource(String name) { for (RemoteFileSourceProvider provider : providers) { - if (!provider.isActive()) { + if (!provider.isActive() || !provider.canSourceResource(name)) { continue; } + try { - if (provider.canSourceResource(name)) { - return new URL(null, "remotefile://" + name, new RemoteFileURLStreamHandler(provider, name)); - } + return new URL(null, "remotefile://" + name, new RemoteFileURLStreamHandler(provider, name)); } catch (java.net.MalformedURLException e) { // Continue to next provider if URL creation fails } } + return super.findResource(name); } @@ -70,11 +100,23 @@ private static class RemoteFileURLStreamHandler extends URLStreamHandler { private final RemoteFileSourceProvider provider; private final String resourceName; + /** + * Constructs a new RemoteFileURLStreamHandler for the specified provider and resource. + * + * @param provider the provider that will source the resource + * @param resourceName the name of the resource to fetch + */ RemoteFileURLStreamHandler(RemoteFileSourceProvider provider, String resourceName) { this.provider = provider; this.resourceName = resourceName; } + /** + * Opens a connection to the resource referenced by this URL. + * + * @param url the URL to open a connection to + * @return a URLConnection to the specified URL + */ @Override protected URLConnection openConnection(URL url) { return new RemoteFileURLConnection(url, provider, resourceName); @@ -89,12 +131,27 @@ private static class RemoteFileURLConnection extends URLConnection { private final String resourceName; private byte[] content; + /** + * Constructs a new RemoteFileURLConnection for the specified URL, provider, and resource. + * + * @param url the URL to connect to + * @param provider the provider that will source the resource + * @param resourceName the name of the resource to fetch + */ RemoteFileURLConnection(URL url, RemoteFileSourceProvider provider, String resourceName) { super(url); this.provider = provider; this.resourceName = resourceName; } + /** + * Opens a connection to the resource by requesting it from the provider. + * + *

This method fetches the resource bytes from the provider with a timeout of + * {@value #RESOURCE_TIMEOUT_SECONDS} seconds. If already connected, this method does nothing. + * + * @throws IOException if the connection fails or times out + */ @Override public void connect() throws IOException { if (!connected) { @@ -109,6 +166,14 @@ public void connect() throws IOException { } } + /** + * Returns an input stream that reads from this connection's resource. + * + *

This method ensures the connection is established before returning the stream. + * + * @return an input stream that reads from the resource + * @throws IOException if the connection cannot be established or if the resource has no content + */ @Override public InputStream getInputStream() throws IOException { connect(); From bf474950b039e5660abdef27a07861cfad183608 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Fri, 2 Jan 2026 10:26:18 -0600 Subject: [PATCH 37/57] Sorted members (#DH-20578) --- .../engine/util/RemoteFileSourceProvider.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java index 02e5eee5feb..377172ea605 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java @@ -20,18 +20,18 @@ public interface RemoteFileSourceProvider { boolean canSourceResource(String resourceName); /** - * Request a resource from the remote source. + * Check if this provider is currently active and should be used for resource requests. * - * @param resourceName the name of the resource to fetch (e.g., "com/example/MyClass.groovy") - * @return a CompletableFuture containing the resource bytes, or null if not found + * @return true if this provider is active, false otherwise */ - CompletableFuture requestResource(String resourceName); + boolean isActive(); /** - * Check if this provider is currently active and should be used for resource requests. + * Request a resource from the remote source. * - * @return true if this provider is active, false otherwise + * @param resourceName the name of the resource to fetch (e.g., "com/example/MyClass.groovy") + * @return a CompletableFuture containing the resource bytes, or null if not found */ - boolean isActive(); + CompletableFuture requestResource(String resourceName); } From 803a9ce1ccadce3ab61053fe623c369c09c5f502 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Fri, 2 Jan 2026 11:49:46 -0600 Subject: [PATCH 38/57] Cleanup RemoteFileSourceCommandResolver (#DH-20578) --- .../RemoteFileSourceCommandResolver.java | 101 ++++++++++++------ 1 file changed, 70 insertions(+), 31 deletions(-) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java index c1d973609af..149edf63b13 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java @@ -20,7 +20,6 @@ import io.deephaven.server.session.TicketRouter; import io.deephaven.server.session.WantsTicketRouter; -import io.grpc.Status; import io.grpc.StatusRuntimeException; import org.apache.arrow.flight.impl.Flight; import org.jetbrains.annotations.Nullable; @@ -62,7 +61,7 @@ private static RemoteFileSourcePluginFetchRequest parseFetchRequest(final Any co * @param data the ByteString data to parse * @return the parsed Any message, or null if parsing fails */ - private static Any parseOrNull(final ByteString data) { + private static Any parseAsAnyOrNull(final ByteString data) { try { return Any.parseFrom(data); } catch (final InvalidProtocolBufferException e) { @@ -71,11 +70,11 @@ private static Any parseOrNull(final ByteString data) { } /** - * Exports a PluginMarker singleton based on the fetch request. + * Exports the PluginMarker singleton based on the fetch request. * The marker object is exported to the session using the result ticket specified in the request, * and flight info is returned containing the endpoint for accessing it. * - * Note: This exports PluginMarker.INSTANCE as a trusted marker. Plugin-specific routing + *

Note: This exports a PluginMarker for the specified plugin name. Plugin-specific routing * is handled by TypedTicket.type in the ConnectRequest phase, which is validated against * the plugin's name() method. * @@ -83,7 +82,7 @@ private static Any parseOrNull(final ByteString data) { * @param descriptor the flight descriptor containing the command * @param request the parsed RemoteFileSourcePluginFetchRequest containing the result ticket * @return a FlightInfo export object containing the plugin endpoint information - * @throws StatusRuntimeException if the request doesn't contain a valid result ID ticket + * @throws StatusRuntimeException if the request doesn't contain a valid result ID ticket or plugin name */ private static SessionState.ExportObject fetchPlugin(@Nullable final SessionState session, final Flight.FlightDescriptor descriptor, @@ -91,23 +90,20 @@ private static SessionState.ExportObject fetchPlugin(@Nullabl final Ticket resultTicket = request.getResultId(); final boolean hasResultId = !resultTicket.getTicket().isEmpty(); if (!hasResultId) { - throw new StatusRuntimeException(Status.INVALID_ARGUMENT - .withDescription("RemoteFileSourcePluginFetchRequest must contain a valid result_id")); + throw Exceptions.statusRuntimeException(Code.INVALID_ARGUMENT, + "RemoteFileSourcePluginFetchRequest must contain a valid result_id"); } final String pluginName = request.getPluginName(); if (pluginName.isEmpty()) { - throw new StatusRuntimeException(Status.INVALID_ARGUMENT - .withDescription("RemoteFileSourcePluginFetchRequest must contain a valid plugin_name")); + throw Exceptions.statusRuntimeException(Code.INVALID_ARGUMENT, + "RemoteFileSourcePluginFetchRequest must contain a valid plugin_name"); } - final SessionState.ExportBuilder markerExportBuilder = - session.newExport(resultTicket, "RemoteFileSourcePluginFetchRequest.resultTicket"); - - // Get singleton marker for this plugin name - // This ensures isType() routing works correctly when multiple plugins use PluginMarker - final SessionState.ExportObject markerExport = - markerExportBuilder.submit(() -> PluginMarker.forPluginName(pluginName)); + // Export a plugin-specific PluginMarker. Plugins using PluginMarker should check + // marker.getPluginName() in isType() to prevent conflicts when multiple plugins share PluginMarker. + session.newExport(resultTicket, "RemoteFileSourcePluginFetchRequest.resultTicket") + .submit(() -> PluginMarker.forPluginName(pluginName)); final Flight.FlightInfo flightInfo = Flight.FlightInfo.newBuilder() .setFlightDescriptor(descriptor) @@ -119,6 +115,7 @@ private static SessionState.ExportObject fetchPlugin(@Nullabl .setTotalRecords(-1) .setTotalBytes(-1) .build(); + return SessionState.wrapAsExport(flightInfo); } @@ -139,88 +136,130 @@ public SessionState.ExportObject flightInfoFor(@Nullable fina final Flight.FlightDescriptor descriptor, final String logId) { if (session == null) { - throw new StatusRuntimeException(Status.UNAUTHENTICATED); + throw Exceptions.statusRuntimeException(Code.UNAUTHENTICATED, + "Could not resolve '" + logId + "': no session available"); } - final Any request = parseOrNull(descriptor.getCmd()); + final Any request = parseAsAnyOrNull(descriptor.getCmd()); if (request == null) { log.error().append("Could not parse remotefilesource command.").endl(); throw Exceptions.statusRuntimeException(Code.INVALID_ARGUMENT, "Could not parse remotefilesource command Any."); } - if (FETCH_PLUGIN_TYPE_URL.equals(request.getTypeUrl())) { - return fetchPlugin(session, descriptor, parseFetchRequest(request)); + if (!FETCH_PLUGIN_TYPE_URL.equals(request.getTypeUrl())) { + log.error().append("Invalid remotefilesource command typeUrl: " + request.getTypeUrl()).endl(); + throw Exceptions.statusRuntimeException(Code.INVALID_ARGUMENT, "Invalid typeUrl: " + request.getTypeUrl()); } - log.error().append("Invalid pivot command typeUrl: " + request.getTypeUrl()).endl(); - throw Exceptions.statusRuntimeException(Code.INVALID_ARGUMENT, "Invalid typeUrl: " + request.getTypeUrl()); + return fetchPlugin(session, descriptor, parseFetchRequest(request)); } + /** + * Visits all flight info that this resolver exposes. + * + *

Not implemented: This resolver does not expose any flight info via list flights. + */ @Override public void forAllFlightInfo(@Nullable final SessionState session, final Consumer visitor) { // nothing to do } + /** + * Returns a log-friendly name for the given ticket. + * + * @throws UnsupportedOperationException always, as this resolver does not support ticket-based routing + */ @Override public String getLogNameFor(final ByteBuffer ticket, final String logId) { - // no tickets + // no ticket-based routing throw new UnsupportedOperationException(); } + /** + * Determines if this resolver is responsible for handling the given command descriptor. + * This resolver handles commands with type URL matching RemoteFileSourcePluginFetchRequest. + * + * @param descriptor the flight descriptor containing the command + * @return true if this resolver handles the command, false otherwise + */ @Override public boolean handlesCommand(final Flight.FlightDescriptor descriptor) { - // If not CMD, there is an error with io.deephaven.server.session.TicketRouter.getPathResolver / handlesPath Assert.eq(descriptor.getType(), "descriptor.getType()", Flight.FlightDescriptor.DescriptorType.CMD, "CMD"); - // No good way to check if this is a valid command without parsing to Any first. - final Any command = parseOrNull(descriptor.getCmd()); + final Any command = parseAsAnyOrNull(descriptor.getCmd()); if (command == null) { return false; } - // Check if the command matches any types that this resolver handles. return FETCH_PLUGIN_TYPE_URL.equals(command.getTypeUrl()); } + /** + * Publishes an export to the session using a ticket. + * + * @throws UnsupportedOperationException always, as this resolver does not support publishing + */ @Override public SessionState.ExportBuilder publish(final SessionState session, final ByteBuffer ticket, final String logId, @Nullable final Runnable onPublish) { - // no publishing throw new UnsupportedOperationException(); } + /** + * Publishes an export to the session using a flight descriptor. + * + * @throws UnsupportedOperationException always, as this resolver does not support publishing + */ @Override public SessionState.ExportBuilder publish(final SessionState session, final Flight.FlightDescriptor descriptor, final String logId, @Nullable final Runnable onPublish) { - // no publishing throw new UnsupportedOperationException(); } + /** + * Resolves a flight descriptor to an export object. + * + * @throws UnsupportedOperationException always, use flightInfoFor() instead + */ @Override public SessionState.ExportObject resolve(@Nullable final SessionState session, final Flight.FlightDescriptor descriptor, final String logId) { - // use flightInfoFor() instead of resolve() for descriptor handling throw new UnsupportedOperationException(); } + /** + * Resolves a ticket to an export object. + * + * @throws UnsupportedOperationException always, as this resolver does not support ticket-based routing + */ @Override public SessionState.ExportObject resolve(@Nullable final SessionState session, final ByteBuffer ticket, final String logId) { - // no tickets throw new UnsupportedOperationException(); } + /** + * Sets the ticket router for this resolver. + * + *

Not implemented: This resolver does not need access to the ticket router. + */ @Override public void setTicketRouter(TicketRouter ticketRouter) { // not needed } + /** + * Returns the ticket route byte for this resolver. + * This resolver does not use ticket-based routing, so returns 0. + * + * @return 0, indicating no ticket routing + */ @Override public byte ticketRoute() { return 0; From 47c1ca280835b9187b40af066f0acaed83b5e4d5 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Fri, 2 Jan 2026 15:11:13 -0600 Subject: [PATCH 39/57] Cleanup RemoteFileSourceMessageStream (#DH-20578) --- .../RemoteFileSourceMessageStream.java | 296 ++++++++++-------- .../proto/remotefilesource.proto | 3 - 2 files changed, 158 insertions(+), 141 deletions(-) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java index 4161d4a937d..85ed23d823c 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java @@ -4,37 +4,42 @@ package io.deephaven.remotefilesource; import com.google.protobuf.InvalidProtocolBufferException; +import io.deephaven.engine.util.RemoteFileSourceClassLoader; +import io.deephaven.engine.util.RemoteFileSourceProvider; import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; import io.deephaven.plugin.type.ObjectCommunicationException; import io.deephaven.plugin.type.ObjectType; import io.deephaven.proto.backplane.grpc.RemoteFileSourceClientRequest; +import io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaRequest; import io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaResponse; import io.deephaven.proto.backplane.grpc.RemoteFileSourceServerRequest; import io.deephaven.proto.backplane.grpc.SetExecutionContextResponse; import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; /** * Message stream implementation for RemoteFileSource bidirectional communication. * Each instance represents a file source provider for one client connection and implements - * RemoteFileSourceProvider so it can be registered with the ClassLoader. + * RemoteFileSourceProvider so it can be registered with the RemoteFileSourceClassLoader. * Only one MessageStream can be "active" at a time (determined by the execution context). - * The ClassLoader checks isActive() on each registered provider to find the active one. + * The RemoteFileSourceClassLoader checks isActive() on each registered provider to find the active one. */ -public class RemoteFileSourceMessageStream implements ObjectType.MessageStream, io.deephaven.engine.util.RemoteFileSourceProvider { +public class RemoteFileSourceMessageStream implements ObjectType.MessageStream, RemoteFileSourceProvider { private static final Logger log = LoggerFactory.getLogger(RemoteFileSourceMessageStream.class); /** * The current execution context containing the active message stream and configuration. * Null when no execution context is active. - * This is accessed by RemoteFileSourcePlugin.PROVIDER to route resource requests to the - * currently active message stream. + * Used by this class's isActive() and canSourceResource() methods to determine if this + * provider should handle resource requests from RemoteFileSourceClassLoader. */ private static volatile RemoteFileSourceExecutionContext executionContext; @@ -42,40 +47,48 @@ public class RemoteFileSourceMessageStream implements ObjectType.MessageStream, private final ObjectType.MessageStream connection; private final Map> pendingRequests = new ConcurrentHashMap<>(); + /** + * Creates a new RemoteFileSourceMessageStream for the given connection. + * Automatically registers this instance as a provider with the RemoteFileSourceClassLoader. + * + * @param connection the message stream connection to the client + */ public RemoteFileSourceMessageStream(final ObjectType.MessageStream connection) { this.connection = connection; - // Register this instance as a provider with the ClassLoader + // Register this instance as a provider with the RemoteFileSourceClassLoader registerWithClassLoader(); } - // RemoteFileSourceProvider interface implementation - each instance is a provider - + /** + * Determines if this provider can source the specified resource. + * Only returns true if this message stream is active, the resource is a .groovy file, + * and the resource path matches one of the configured resource paths. + * + * @param resourcePath the path of the resource to check + * @return true if this provider can source the resource, false otherwise + */ @Override - public boolean canSourceResource(String resourceName) { + public boolean canSourceResource(String resourcePath) { // Only active if this instance is the currently active message stream if (!isActive()) { return false; } // Only handle .groovy source files, not compiled .class files - if (!resourceName.endsWith(".groovy")) { + if (!resourcePath.endsWith(".groovy")) { return false; } RemoteFileSourceExecutionContext context = executionContext; - if (context == null || context.getActiveMessageStream() != this) { - return false; - } - java.util.List resourcePaths = context.getResourcePaths(); + List resourcePaths = context.getResourcePaths(); if (resourcePaths.isEmpty()) { return false; } - // Resource names from ClassLoader always use forward slashes, not backslashes for (String contextResourcePath : resourcePaths) { - if (resourceName.equals(contextResourcePath)) { - log.info().append("✅ Can source: ").append(resourceName).endl(); + if (resourcePath.equals(contextResourcePath)) { + log.info().append("Can source: ").append(resourcePath).endl(); return true; } } @@ -83,38 +96,46 @@ public boolean canSourceResource(String resourceName) { return false; } + /** + * Requests a resource from the remote client. + * Sends a request to the client and returns a future that will be completed when the client responds. + * Only services requests if this message stream is active. + * + * @param resourcePath the name of the resource to request + * @return a CompletableFuture that will contain the resource bytes when available, or null if inactive + */ @Override - public java.util.concurrent.CompletableFuture requestResource(String resourceName) { + public CompletableFuture requestResource(String resourcePath) { // Only service requests if this instance is active if (!isActive()) { - log.warn().append("Request for resource ").append(resourceName) + log.warn().append("Request for resource ").append(resourcePath) .append(" on inactive message stream").endl(); - return java.util.concurrent.CompletableFuture.completedFuture(null); + return CompletableFuture.completedFuture(null); } - log.info().append("📥 Requesting resource: ").append(resourceName).endl(); + log.info().append("Requesting resource: ").append(resourcePath).endl(); - String requestId = java.util.UUID.randomUUID().toString(); - java.util.concurrent.CompletableFuture future = new java.util.concurrent.CompletableFuture<>(); + String requestId = UUID.randomUUID().toString(); + CompletableFuture future = new CompletableFuture<>(); pendingRequests.put(requestId, future); try { // Build RemoteFileSourceMetaRequest proto - io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaRequest metaRequest = - io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaRequest.newBuilder() - .setResourceName(resourceName) + RemoteFileSourceMetaRequest metaRequest = + RemoteFileSourceMetaRequest.newBuilder() + .setResourceName(resourcePath) .build(); // Wrap in RemoteFileSourceServerRequest (server→client) - io.deephaven.proto.backplane.grpc.RemoteFileSourceServerRequest message = - io.deephaven.proto.backplane.grpc.RemoteFileSourceServerRequest.newBuilder() + RemoteFileSourceServerRequest message = + RemoteFileSourceServerRequest.newBuilder() .setRequestId(requestId) .setMetaRequest(metaRequest) .build(); - java.nio.ByteBuffer buffer = java.nio.ByteBuffer.wrap(message.toByteArray()); + ByteBuffer buffer = ByteBuffer.wrap(message.toByteArray()); - log.info().append("Sending resource request for: ").append(resourceName) + log.info().append("Sending resource request for: ").append(resourcePath) .append(" with requestId: ").append(requestId).endl(); connection.onData(buffer); @@ -126,30 +147,34 @@ public java.util.concurrent.CompletableFuture requestResource(String res return future; } + /** + * Checks if this message stream is currently active. + * A message stream is active when the execution context is set and this instance is the active stream. + * + * @return true if this message stream is active, false otherwise + */ @Override public boolean isActive() { RemoteFileSourceExecutionContext context = executionContext; return context != null && context.getActiveMessageStream() == this; } - // Static methods for execution context management - /** * Sets the execution context with the active message stream and resource paths. * This should be called when a script execution begins. * - * @param messageStream the message stream to set as active (must not be null) - * @param packages list of resource paths to resolve from remote source - * @throws IllegalArgumentException if messageStream is null (use clearExecutionContext() instead) + * @param messageStream the message stream to set as active + * @param resourcePaths list of resource paths to resolve from remote source + * @throws IllegalArgumentException if messageStream is null */ - public static void setExecutionContext(RemoteFileSourceMessageStream messageStream, java.util.List packages) { + public static void setExecutionContext(RemoteFileSourceMessageStream messageStream, List resourcePaths) { if (messageStream == null) { - throw new IllegalArgumentException("messageStream must not be null. Use clearExecutionContext() to clear the context."); + throw new IllegalArgumentException("messageStream must not be null"); } - executionContext = new RemoteFileSourceExecutionContext(messageStream, packages); + executionContext = new RemoteFileSourceExecutionContext(messageStream, resourcePaths); log.info().append("Set execution context with ") - .append(packages != null ? packages.size() : 0).append(" resource paths").endl(); + .append(executionContext.getResourcePaths().size()).append(" resource paths").endl(); } /** @@ -171,70 +196,26 @@ public static RemoteFileSourceExecutionContext getExecutionContext() { return executionContext; } - // Instance methods for MessageStream implementation - + /** + * Handles incoming data from the client. + * Parses RemoteFileSourceClientRequest messages and processes meta responses + * or execution context updates from the client. + * + * @param payload the message payload containing the protobuf data + * @param references optional references (not used) + * @throws ObjectCommunicationException if the message cannot be parsed + */ @Override public void onData(ByteBuffer payload, Object... references) throws ObjectCommunicationException { try { - // Parse as RemoteFileSourceClientRequest proto (client→server) byte[] bytes = new byte[payload.remaining()]; payload.get(bytes); RemoteFileSourceClientRequest message = RemoteFileSourceClientRequest.parseFrom(bytes); - String requestId = message.getRequestId(); - if (message.hasMetaResponse()) { - // Client is responding to a resource request - RemoteFileSourceMetaResponse response = message.getMetaResponse(); - - CompletableFuture future = pendingRequests.remove(requestId); - if (future != null) { - byte[] content = response.getContent().toByteArray(); - - log.info().append("Received resource response for requestId: ").append(requestId) - .append(", found: ").append(response.getFound()) - .append(", content length: ").append(content.length).endl(); - - if (!response.getError().isEmpty()) { - log.warn().append("Error in response: ").append(response.getError()).endl(); - } - - future.complete(content); - } else { - log.warn().append("Received response for unknown requestId: ").append(requestId).endl(); - } - } else if (message.hasTestCommand()) { - // Client sent a test command - String command = message.getTestCommand(); - log.info().append("Received test command from client: ").append(command).endl(); - - if (command.startsWith("TEST:")) { - String resourceName = command.substring(5).trim(); - log.info().append("Client initiated test for resource: ").append(resourceName).endl(); - testRequestResource(resourceName); - } + handleMetaResponse(message.getRequestId(), message.getMetaResponse()); } else if (message.hasSetExecutionContext()) { - // Client is requesting this message stream to become active - java.util.List packages = message.getSetExecutionContext().getResourcePathsList(); - setExecutionContext(this, packages); - log.info().append("Client set execution context for this message stream with ") - .append(packages.size()).append(" resource paths").endl(); - - // Send acknowledgment back to client - SetExecutionContextResponse response = SetExecutionContextResponse.newBuilder() - .setSuccess(true) - .build(); - - RemoteFileSourceServerRequest serverRequest = RemoteFileSourceServerRequest.newBuilder() - .setRequestId(requestId) - .setSetExecutionContextResponse(response) - .build(); - - try { - connection.onData(ByteBuffer.wrap(serverRequest.toByteArray())); - } catch (ObjectCommunicationException e) { - log.error().append("Failed to send execution context acknowledgment: ").append(e).endl(); - } + handleSetExecutionContext(message.getRequestId(), message.getSetExecutionContext().getResourcePathsList()); } else { log.warn().append("Received unknown message type from client").endl(); } @@ -244,14 +225,80 @@ public void onData(ByteBuffer payload, Object... references) throws ObjectCommun } } + /** + * Handles a meta response from the client containing requested resource content. + * + * @param requestId the request ID + * @param response the meta response from the client + */ + private void handleMetaResponse(String requestId, RemoteFileSourceMetaResponse response) { + CompletableFuture future = pendingRequests.remove(requestId); + if (future == null) { + log.warn().append("Received response for unknown requestId: ").append(requestId).endl(); + return; + } + + byte[] content = response.getContent().toByteArray(); + + log.info().append("Received resource response for requestId: ").append(requestId) + .append(", found: ").append(response.getFound()) + .append(", content length: ").append(content.length).endl(); + + if (!response.getError().isEmpty()) { + log.warn().append("Error in response: ").append(response.getError()).endl(); + } + + future.complete(content); + } + + /** + * Handles a request from the client to set the execution context. + * + * @param requestId the request ID + * @param resourcePaths the list of resource paths to resolve from remote source + */ + private void handleSetExecutionContext(String requestId, List resourcePaths) { + setExecutionContext(this, resourcePaths); + log.info().append("Client set execution context for this message stream with ") + .append(resourcePaths.size()).append(" resource paths").endl(); + + sendExecutionContextAcknowledgment(requestId); + } + + /** + * Sends an acknowledgment to the client that the execution context was successfully set. + * + * @param requestId the request ID to acknowledge + */ + private void sendExecutionContextAcknowledgment(String requestId) { + SetExecutionContextResponse response = SetExecutionContextResponse.newBuilder() + .setSuccess(true) + .build(); + + RemoteFileSourceServerRequest serverRequest = RemoteFileSourceServerRequest.newBuilder() + .setRequestId(requestId) + .setSetExecutionContextResponse(response) + .build(); + + try { + connection.onData(ByteBuffer.wrap(serverRequest.toByteArray())); + } catch (ObjectCommunicationException e) { + log.error().append("Failed to send execution context acknowledgment: ").append(e).endl(); + } + } + + /** + * Handles cleanup when the message stream is closed. + * Unregisters this provider from the RemoteFileSourceClassLoader, clears the execution context if this was active, + * and cancels all pending resource requests. + */ @Override public void onClose() { - // Unregister this provider from the ClassLoader + // Unregister this provider from the RemoteFileSourceClassLoader unregisterFromClassLoader(); // Clear execution context if this was the active stream - RemoteFileSourceExecutionContext context = executionContext; - if (context != null && context.getActiveMessageStream() == this) { + if (isActive()) { clearExecutionContext(); } @@ -261,58 +308,31 @@ public void onClose() { } /** - * Register this message stream instance as a provider with the ClassLoader. + * Register this message stream instance as a provider with the RemoteFileSourceClassLoader. */ private void registerWithClassLoader() { - io.deephaven.engine.util.RemoteFileSourceClassLoader classLoader = - io.deephaven.engine.util.RemoteFileSourceClassLoader.getInstance(); + RemoteFileSourceClassLoader classLoader = RemoteFileSourceClassLoader.getInstance(); if (classLoader != null) { classLoader.registerProvider(this); - log.info().append("✅ Registered RemoteFileSourceMessageStream provider with ClassLoader").endl(); + log.info().append("Registered RemoteFileSourceMessageStream provider with RemoteFileSourceClassLoader").endl(); } else { - log.warn().append("⚠️ RemoteFileSourceClassLoader not available").endl(); + log.warn().append("RemoteFileSourceClassLoader not available").endl(); } } /** - * Unregister this message stream instance from the ClassLoader. + * Unregister this message stream instance from the RemoteFileSourceClassLoader. */ private void unregisterFromClassLoader() { - io.deephaven.engine.util.RemoteFileSourceClassLoader classLoader = - io.deephaven.engine.util.RemoteFileSourceClassLoader.getInstance(); + RemoteFileSourceClassLoader classLoader = RemoteFileSourceClassLoader.getInstance(); if (classLoader != null) { classLoader.unregisterProvider(this); - log.info().append("🔴 Unregistered RemoteFileSourceMessageStream provider from ClassLoader").endl(); + log.info().append("Unregistered RemoteFileSourceMessageStream provider from RemoteFileSourceClassLoader").endl(); } } - /** - * Test method to request a resource and log the result. This can be called from the server console to test the - * bidirectional communication. - * - * @param resourceName the resource to request - */ - public void testRequestResource(String resourceName) { - log.info().append("Testing resource request for: ").append(resourceName).endl(); - - requestResource(resourceName) - .orTimeout(30, TimeUnit.SECONDS) - .whenComplete((content, error) -> { - if (error != null) { - log.error().append("Error requesting resource ").append(resourceName) - .append(": ").append(error).endl(); - } else { - log.info().append("Successfully received resource ").append(resourceName) - .append(" (").append(content.length).append(" bytes)").endl(); - if (content.length > 0 && content.length < 1000) { - String contentStr = new String(content, StandardCharsets.UTF_8); - log.info().append("Resource content:\n").append(contentStr).endl(); - } - } - }); - } /** * Encapsulates the execution context for remote file source operations. @@ -322,7 +342,7 @@ public void testRequestResource(String resourceName) { */ public static class RemoteFileSourceExecutionContext { private final RemoteFileSourceMessageStream activeMessageStream; - private final java.util.List resourcePaths; + private final List resourcePaths; /** * Creates a new execution context. @@ -331,9 +351,9 @@ public static class RemoteFileSourceExecutionContext { * @param resourcePaths list of resource paths to resolve from remote source */ public RemoteFileSourceExecutionContext(RemoteFileSourceMessageStream activeMessageStream, - java.util.List resourcePaths) { + List resourcePaths) { this.activeMessageStream = activeMessageStream; - this.resourcePaths = resourcePaths != null ? resourcePaths : java.util.Collections.emptyList(); + this.resourcePaths = resourcePaths != null ? resourcePaths : Collections.emptyList(); } /** @@ -350,8 +370,8 @@ public RemoteFileSourceMessageStream getActiveMessageStream() { * * @return a copy of the list of resource paths */ - public java.util.List getResourcePaths() { - return new java.util.ArrayList<>(resourcePaths); + public List getResourcePaths() { + return new ArrayList<>(resourcePaths); } } } diff --git a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto index b383a776e6b..26cc68c9986 100644 --- a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto +++ b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto @@ -36,9 +36,6 @@ message RemoteFileSourceClientRequest { // Set the execution context ID for script execution SetExecutionContextRequest set_execution_context = 3; - - // Test command (e.g., "TEST:com/example/Test.java") - client triggers server to request a resource back - string test_command = 4; } } From ac126afa7290ef55e5329a412e5ba0be8ebca70e Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Fri, 2 Jan 2026 15:23:36 -0600 Subject: [PATCH 40/57] Cleanup RemoteFileSourcePlugin (#DH-20578) --- .../RemoteFileSourcePlugin.java | 28 +++++++++---------- .../JsRemoteFileSourceService.java | 2 +- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java index 09af2b3430f..92107412fba 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java @@ -4,8 +4,6 @@ package io.deephaven.remotefilesource; import com.google.auto.service.AutoService; -import io.deephaven.internal.log.LoggerFactory; -import io.deephaven.io.logger.Logger; import io.deephaven.plugin.type.ObjectType; import io.deephaven.plugin.type.ObjectTypeBase; import io.deephaven.plugin.type.ObjectCommunicationException; @@ -14,24 +12,24 @@ import java.nio.ByteBuffer; /** - * ObjectType plugin for RemoteFileSource. This plugin is registered via @AutoService - * and handles creation of RemoteFileSourceMessageStream connections. - * - * This plugin uses a PluginMarker with a type field instead of instanceof checks, - * allowing it to work across language boundaries (Java/Python). The Flight command - * creates a PluginMarker with type="RemoteFileSource" which this plugin recognizes. - * + * ObjectType plugin for remote file sources. This plugin is registered via @AutoService + * and handles creation of RemoteFileSourceMessageStream connections for bidirectional + * communication with clients. + *

+ * This plugin recognizes PluginMarker objects whose pluginName matches this plugin's name. + * When a connection is established, a RemoteFileSourceMessageStream is created to handle + * bidirectional message passing between client and server. + *

* Each RemoteFileSourceMessageStream instance registers itself as a provider with the - * ClassLoader when created and unregisters when closed. The ClassLoader checks isActive() - * on each registered provider to find the currently active one. + * RemoteFileSourceClassLoader when created and unregisters when closed. The RemoteFileSourceClassLoader + * uses isActive() to determine which registered provider should handle resource requests. */ @AutoService(ObjectType.class) public class RemoteFileSourcePlugin extends ObjectTypeBase { - private static final Logger log = LoggerFactory.getLogger(RemoteFileSourcePlugin.class); @Override public String name() { - return "DeephavenGroovyRemoteFileSourcePlugin"; + return "DeephavenRemoteFileSourcePlugin"; } @Override @@ -50,10 +48,10 @@ public MessageStream compatibleClientConnection(Object object, MessageStream con throw new ObjectCommunicationException("Expected RemoteFileSource marker object, got " + object.getClass()); } + // Send initial empty message to client as required by the ObjectType contract connection.onData(ByteBuffer.allocate(0)); - // Create and return a new message stream for this connection - // All the logic is in the static RemoteFileSourceMessageStream class + // Return a new bidirectional message stream for this connection return new RemoteFileSourceMessageStream(connection); } } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index dd0d6ef0f09..f5bc526cca1 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -104,7 +104,7 @@ private static Promise fetchPluginFlightInfo(WorkerConnection connec */ @JsIgnore public static Promise fetchPlugin(@TsTypeRef(Object.class) WorkerConnection connection) { - String pluginName = "DeephavenGroovyRemoteFileSourcePlugin"; + String pluginName = "DeephavenRemoteFileSourcePlugin"; return fetchPluginFlightInfo(connection, pluginName) .then(flightInfo -> { // The first endpoint contains the ticket for the plugin instance. From 23b14f9920cc870f47de3625a5c5aceceb14de4d Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Fri, 2 Jan 2026 15:25:15 -0600 Subject: [PATCH 41/57] Cleanup RemoteFileSourceTicketResolverFactoryService (#DH-20578) --- .../RemoteFileSourceTicketResolverFactoryService.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceTicketResolverFactoryService.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceTicketResolverFactoryService.java index fa398c85d95..204161b28a4 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceTicketResolverFactoryService.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceTicketResolverFactoryService.java @@ -7,6 +7,11 @@ import io.deephaven.server.runner.TicketResolversFromServiceLoader; import io.deephaven.server.session.TicketResolver; +/** + * Factory service for creating RemoteFileSourceCommandResolver instances. + * This service is registered via @AutoService and provides ticket resolver functionality + * for handling remote file source plugin commands through the Deephaven server infrastructure. + */ @AutoService(TicketResolversFromServiceLoader.Factory.class) public class RemoteFileSourceTicketResolverFactoryService implements TicketResolversFromServiceLoader.Factory { @Override From 440bd2327dcb78d21a5f831e21cf21216102401c Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Fri, 2 Jan 2026 15:35:06 -0600 Subject: [PATCH 42/57] Cleanup PluginMarker (#DH-20578) --- .../main/java/io/deephaven/plugin/type/PluginMarker.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java b/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java index 570af422ecc..158ff2d0a65 100644 --- a/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java +++ b/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java @@ -3,16 +3,21 @@ // package io.deephaven.plugin.type; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + /** * A generic marker object for plugin exports that can be shared across multiple plugin types. + *

* IMPORTANT: The pluginName field is required because ObjectTypeLookup.findObjectType() * returns the FIRST plugin where isType() returns true. Without plugin-specific identification * in isType(), multiple plugins using PluginMarker would conflict, and whichever is registered * first would intercept all PluginMarker instances. + *

* This class uses a singleton pattern - one instance per pluginName. */ public class PluginMarker { - private static final java.util.Map INSTANCES = new java.util.concurrent.ConcurrentHashMap<>(); + private static final Map INSTANCES = new ConcurrentHashMap<>(); private final String pluginName; From 9e74429ef464f6c0d259283c25754efb00beeb25 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Fri, 2 Jan 2026 15:44:13 -0600 Subject: [PATCH 43/57] Changed back to resourceName convention to match class loaders (#DH-20578) --- .../RemoteFileSourceMessageStream.java | 22 +++++++++---------- .../proto/remotefilesource.proto | 6 ++--- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java index 85ed23d823c..9cc3879b80f 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java @@ -64,18 +64,18 @@ public RemoteFileSourceMessageStream(final ObjectType.MessageStream connection) * Only returns true if this message stream is active, the resource is a .groovy file, * and the resource path matches one of the configured resource paths. * - * @param resourcePath the path of the resource to check + * @param resourceName the name of the resource to check * @return true if this provider can source the resource, false otherwise */ @Override - public boolean canSourceResource(String resourcePath) { + public boolean canSourceResource(String resourceName) { // Only active if this instance is the currently active message stream if (!isActive()) { return false; } // Only handle .groovy source files, not compiled .class files - if (!resourcePath.endsWith(".groovy")) { + if (!resourceName.endsWith(".groovy")) { return false; } @@ -87,8 +87,8 @@ public boolean canSourceResource(String resourcePath) { } for (String contextResourcePath : resourcePaths) { - if (resourcePath.equals(contextResourcePath)) { - log.info().append("Can source: ").append(resourcePath).endl(); + if (resourceName.equals(contextResourcePath)) { + log.info().append("Can source: ").append(resourceName).endl(); return true; } } @@ -101,19 +101,19 @@ public boolean canSourceResource(String resourcePath) { * Sends a request to the client and returns a future that will be completed when the client responds. * Only services requests if this message stream is active. * - * @param resourcePath the name of the resource to request + * @param resourceName the name of the resource to request * @return a CompletableFuture that will contain the resource bytes when available, or null if inactive */ @Override - public CompletableFuture requestResource(String resourcePath) { + public CompletableFuture requestResource(String resourceName) { // Only service requests if this instance is active if (!isActive()) { - log.warn().append("Request for resource ").append(resourcePath) + log.warn().append("Request for resource ").append(resourceName) .append(" on inactive message stream").endl(); return CompletableFuture.completedFuture(null); } - log.info().append("Requesting resource: ").append(resourcePath).endl(); + log.info().append("Requesting resource: ").append(resourceName).endl(); String requestId = UUID.randomUUID().toString(); CompletableFuture future = new CompletableFuture<>(); @@ -123,7 +123,7 @@ public CompletableFuture requestResource(String resourcePath) { // Build RemoteFileSourceMetaRequest proto RemoteFileSourceMetaRequest metaRequest = RemoteFileSourceMetaRequest.newBuilder() - .setResourceName(resourcePath) + .setResourceName(resourceName) .build(); // Wrap in RemoteFileSourceServerRequest (server→client) @@ -135,7 +135,7 @@ public CompletableFuture requestResource(String resourcePath) { ByteBuffer buffer = ByteBuffer.wrap(message.toByteArray()); - log.info().append("Sending resource request for: ").append(resourcePath) + log.info().append("Sending resource request for: ").append(resourceName) .append(" with requestId: ").append(requestId).endl(); connection.onData(buffer); diff --git a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto index 26cc68c9986..20cd589fce1 100644 --- a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto +++ b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending + * Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending */ syntax = "proto3"; @@ -34,7 +34,7 @@ message RemoteFileSourceClientRequest { // Response to a resource request RemoteFileSourceMetaResponse meta_response = 2; - // Set the execution context ID for script execution + // Set the execution context for script execution SetExecutionContextRequest set_execution_context = 3; } } @@ -53,7 +53,7 @@ message RemoteFileSourceMetaResponse { // Indicates whether the resource was found bool found = 2; - // Optional: error message if the resource could not be retrieved + // Error message if the resource could not be retrieved string error = 3; } From 140b3ac5ea4b689e24bcce7f1f7dfd2dcecf2dc3 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Fri, 2 Jan 2026 15:52:47 -0600 Subject: [PATCH 44/57] Moved method (#DH-20578) --- .../main/java/io/deephaven/web/client/api/CoreClient.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java b/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java index ecdeee17289..88da02de07c 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java @@ -142,6 +142,10 @@ public Promise onConnected(@JsOptional Double timeoutInMillis) { return ideConnection.onConnected(); } + public Promise getRemoteFileSourceService() { + return JsRemoteFileSourceService.fetchPlugin(ideConnection.connection.get()); + } + public Promise getServerConfigValues() { return getConfigs( c -> ideConnection.connection.get().configServiceClient().getConfigurationConstants( @@ -155,10 +159,6 @@ public JsStorageService getStorageService() { return new JsStorageService(ideConnection.connection.get()); } - public Promise getRemoteFileSourceService() { - return JsRemoteFileSourceService.fetchPlugin(ideConnection.connection.get()); - } - public Promise getAsIdeConnection() { return Promise.resolve(ideConnection); } From 77ff6b33e3c42c765a85e0653aa2f1024410f534 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Fri, 2 Jan 2026 16:00:53 -0600 Subject: [PATCH 45/57] Cleanup JsProtobufUtils (#DH-20578) --- .../web/client/api/JsProtobufUtils.java | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/JsProtobufUtils.java b/web/client-api/src/main/java/io/deephaven/web/client/api/JsProtobufUtils.java index 7413aedd954..2d61a93523b 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/JsProtobufUtils.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/JsProtobufUtils.java @@ -11,6 +11,11 @@ */ public class JsProtobufUtils { + // Varint encoding constants + private static final int VARINT_CONTINUATION_BIT = 0x80; // 10000000 - indicates more bytes follow + private static final int VARINT_DATA_MASK = 0x7F; // 01111111 - extracts 7 bits of data + private static final int VARINT_BYTE_THRESHOLD = 128; // Values >= 128 require multiple bytes + private JsProtobufUtils() { // Utility class, no instantiation } @@ -88,8 +93,8 @@ private static int calculateFieldSize(int tag, int dataLength) { *

  • 2 bytes: 128 to 16,383 (2^14 - 1)
    • *
  • 3 bytes: 16,384 to 2,097,151 (2^21 - 1)
    • *
  • 4 bytes: 2,097,152 to 268,435,455 (2^28 - 1)
    • - *
  • 5 bytes: 268,435,456 to 4,294,967,295 (2^35 - 1, max unsigned 32-bit)
    • - *
  • 10 bytes: negative numbers (due to sign extension)
    • + *
  • 5 bytes: 268,435,456 to 4,294,967,295 (max unsigned 32-bit int)
    • + *
  • 10 bytes: negative numbers (sign-extended to 64 bits in varint encoding)
    • * * * @param value the integer value to encode @@ -97,16 +102,16 @@ private static int calculateFieldSize(int tag, int dataLength) { */ private static int sizeOfVarint(int value) { if (value < 0) - return 10; // Negative numbers use sign extension, always 10 bytes - if (value < 128) // 2^7 + return 10; // Negative numbers use sign extension, always 10 bytes + if (value < VARINT_BYTE_THRESHOLD) // 2^7 = 128 return 1; - if (value < 16384) // 2^14 + if (value < 16384) // 2^14 return 2; - if (value < 2097152) // 2^21 + if (value < 2097152) // 2^21 return 3; - if (value < 268435456) // 2^28 + if (value < 268435456) // 2^28 return 4; - return 5; // 2^35 (max for positive 32-bit int) + return 5; // Max unsigned 32-bit int requires 5 bytes } /** @@ -164,9 +169,9 @@ private static int writeField(Uint8Array buffer, int pos, int tag, Uint8Array da * @return the new position after writing */ private static int writeVarint(Uint8Array buffer, int pos, int value) { - while (value >= 128) { + while (value >= VARINT_BYTE_THRESHOLD) { // Extract lowest 7 bits and set continuation flag (8th bit = 1) - buffer.setAt(pos++, (double) ((value & 0x7F) | 0x80)); + buffer.setAt(pos++, (double) ((value & VARINT_DATA_MASK) | VARINT_CONTINUATION_BIT)); // Shift right by 7 to process next chunk value >>>= 7; // Unsigned right shift to handle large positive values } From df05a6249838e3e94a1fb413479ad083e28964ec Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Fri, 2 Jan 2026 17:07:47 -0600 Subject: [PATCH 46/57] Cleanup JsRemoteFileSourceService (#DH-20578) --- .../JsRemoteFileSourceService.java | 157 +++++++++++++----- 1 file changed, 111 insertions(+), 46 deletions(-) diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index f5bc526cca1..b1070ca3f60 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -43,13 +43,28 @@ /** * JavaScript client for the RemoteFileSource service. Provides bidirectional communication with the server-side - * RemoteFileSourceServicePlugin via a message stream. + * RemoteFileSourcePlugin via a message stream. + *

    + * Events: + *

      + *
    • {@link #EVENT_MESSAGE}: Fired for unrecognized messages from the server
      • + *
    • {@link #EVENT_REQUEST_SOURCE}: Fired when the server requests a resource from the client
      • + *
    */ @JsType(namespace = "dh.remotefilesource", name = "RemoteFileSourceService") public class JsRemoteFileSourceService extends HasEventHandling { + /** Event name for generic messages from the server */ public static final String EVENT_MESSAGE = "message"; + + /** Event name for resource request events from the server */ public static final String EVENT_REQUEST_SOURCE = "requestsource"; + // Plugin name must match RemoteFileSourcePlugin.name() on the server + private static final String PLUGIN_NAME = "DeephavenRemoteFileSourcePlugin"; + + // Timeout for setExecutionContext requests (in milliseconds) + private static final int SET_EXECUTION_CONTEXT_TIMEOUT_MS = 30000; // 30 seconds + private final JsWidget widget; // Track pending setExecutionContext requests @@ -65,18 +80,17 @@ private JsRemoteFileSourceService(JsWidget widget) { * Fetches the FlightInfo for the plugin fetch command. * * @param connection the worker connection to use - * @param pluginName the name of the plugin to fetch * @return a promise that resolves to the FlightInfo for the plugin fetch */ @JsIgnore - private static Promise fetchPluginFlightInfo(WorkerConnection connection, String pluginName) { + private static Promise fetchPluginFlightInfo(WorkerConnection connection) { // Create a new export ticket for the result Ticket resultTicket = connection.getTickets().newExportTicket(); // Create the fetch request RemoteFileSourcePluginFetchRequest fetchRequest = new RemoteFileSourcePluginFetchRequest(); fetchRequest.setResultId(resultTicket); - fetchRequest.setPluginName(pluginName); + fetchRequest.setPluginName(PLUGIN_NAME); // Serialize the request to bytes Uint8Array innerRequestBytes = fetchRequest.serializeBinary(); @@ -92,8 +106,8 @@ private static Promise fetchPluginFlightInfo(WorkerConnection connec descriptor.setCmd(anyWrappedBytes); // Send the getFlightInfo request - return Callbacks.grpcUnaryPromise( - c -> connection.flightServiceClient().getFlightInfo(descriptor, connection.metadata(), c::apply)); + return Callbacks.grpcUnaryPromise(c -> + connection.flightServiceClient().getFlightInfo(descriptor, connection.metadata(), c::apply)); } /** @@ -104,8 +118,7 @@ private static Promise fetchPluginFlightInfo(WorkerConnection connec */ @JsIgnore public static Promise fetchPlugin(@TsTypeRef(Object.class) WorkerConnection connection) { - String pluginName = "DeephavenRemoteFileSourcePlugin"; - return fetchPluginFlightInfo(connection, pluginName) + return fetchPluginFlightInfo(connection) .then(flightInfo -> { // The first endpoint contains the ticket for the plugin instance. // This is the standard Flight pattern: we passed resultTicket in the request, @@ -124,14 +137,14 @@ public static Promise fetchPlugin(@TsTypeRef(Object.c // The type must match RemoteFileSourcePlugin.name() TypedTicket typedTicket = new TypedTicket(); typedTicket.setTicket(dhTicket); - typedTicket.setType(pluginName); + typedTicket.setType(PLUGIN_NAME); JsWidget widget = new JsWidget(connection, typedTicket); JsRemoteFileSourceService service = new JsRemoteFileSourceService(widget); return service.connect(); } else { - return Promise.reject("No endpoints returned from RemoteFileSource plugin fetch"); + return Promise.reject("No endpoints returned from " + PLUGIN_NAME + " plugin fetch"); } }); } @@ -143,47 +156,83 @@ public static Promise fetchPlugin(@TsTypeRef(Object.c */ @JsIgnore private Promise connect() { - widget.addEventListener("message", (Event event) -> { - // Parse the message as RemoteFileSourceServerRequest proto (server→client) - Uint8Array payload = event.getDetail().getDataAsU8(); - - try { - RemoteFileSourceServerRequest message = - RemoteFileSourceServerRequest.deserializeBinary(payload); - - if (message.hasMetaRequest()) { - // If server has requested a resource from the client, fire request event - RemoteFileSourceMetaRequest request = message.getMetaRequest(); - - DomGlobal.setTimeout(ignore -> fireEvent(EVENT_REQUEST_SOURCE, - new ResourceRequestEvent(message.getRequestId(), request)), 0); - } else if (message.hasSetExecutionContextResponse()) { - // Server acknowledged execution context was set - String requestId = message.getRequestId(); - Promise.PromiseExecutorCallbackFn.ResolveCallbackFn resolveCallback = - pendingSetExecutionContextRequests.remove(requestId); - if (resolveCallback != null) { - SetExecutionContextResponse response = message.getSetExecutionContextResponse(); - resolveCallback.onInvoke(response.getSuccess()); - } - } else { - // Unknown message type - DomGlobal.setTimeout(ignore -> fireEvent(EVENT_MESSAGE, event.getDetail()), 0); - } - } catch (Exception e) { - // Failed to parse as proto, fire generic message event - DomGlobal.setTimeout(ignore -> fireEvent(EVENT_MESSAGE, event.getDetail()), 0); - } - }); - + widget.addEventListener("message", this::handleMessage); return widget.refetch().then(w -> Promise.resolve(this)); } + /** + * Handles incoming messages from the server. + * + * @param event the message event from the server + */ + @JsIgnore + private void handleMessage(Event event) { + Uint8Array payload = event.getDetail().getDataAsU8(); + + RemoteFileSourceServerRequest message; + try { + message = RemoteFileSourceServerRequest.deserializeBinary(payload); + } catch (Exception e) { + // Failed to parse as proto, fire generic message event + handleUnknownMessage(event); + return; + } + + // Route the parsed message to the appropriate handler + if (message.hasMetaRequest()) { + handleMetaRequest(message); + } else if (message.hasSetExecutionContextResponse()) { + handleSetExecutionContextResponse(message); + } else { + handleUnknownMessage(event); + } + } + + /** + * Handles a meta request (resource request) from the server. + * + * @param message the server request message + */ + @JsIgnore + private void handleMetaRequest(RemoteFileSourceServerRequest message) { + RemoteFileSourceMetaRequest request = message.getMetaRequest(); + DomGlobal.setTimeout(ignore -> fireEvent(EVENT_REQUEST_SOURCE, + new ResourceRequestEvent(message.getRequestId(), request)), 0); + } + + /** + * Handles a set execution context response from the server. + * + * @param message the server request message + */ + @JsIgnore + private void handleSetExecutionContextResponse(RemoteFileSourceServerRequest message) { + String requestId = message.getRequestId(); + Promise.PromiseExecutorCallbackFn.ResolveCallbackFn resolveCallback = + pendingSetExecutionContextRequests.remove(requestId); + if (resolveCallback != null) { + SetExecutionContextResponse response = message.getSetExecutionContextResponse(); + resolveCallback.onInvoke(response.getSuccess()); + } + } + + /** + * Handles an unknown or unparseable message from the server. + * + * @param event the message event + */ + @JsIgnore + private void handleUnknownMessage(Event event) { + DomGlobal.setTimeout(ignore -> fireEvent(EVENT_MESSAGE, event.getDetail()), 0); + } + /** * Sets the execution context on the server to identify this message stream as active * for script execution. * - * @param resourcePaths array of resource paths to resolve from remote source (e.g., ["com/example/Test.groovy", "org/mycompany/Utils.groovy"]) + * @param resourcePaths array of resource paths to resolve from remote source + * (e.g., ["com/example/Test.groovy", "org/mycompany/Utils.groovy"]), + * or null/empty for no specific resources * @return a promise that resolves to true if the server successfully set the execution context, false otherwise */ @JsMethod @@ -195,6 +244,17 @@ public Promise setExecutionContext(@JsOptional String[] resourcePaths) // Store the resolve callback to call when we get the acknowledgment pendingSetExecutionContextRequests.put(requestId, resolve); + // Set a timeout to reject the promise if no response is received + DomGlobal.setTimeout(ignore -> { + Promise.PromiseExecutorCallbackFn.ResolveCallbackFn callback = + pendingSetExecutionContextRequests.remove(requestId); + if (callback != null) { + // Request timed out - reject the promise + reject.onInvoke("setExecutionContext request timed out after " + + SET_EXECUTION_CONTEXT_TIMEOUT_MS + "ms"); + } + }, SET_EXECUTION_CONTEXT_TIMEOUT_MS); + RemoteFileSourceClientRequest clientRequest = getSetExecutionContextRequest(resourcePaths, requestId); sendClientRequest(clientRequest); }); @@ -232,7 +292,8 @@ private void sendClientRequest(RemoteFileSourceClientRequest clientRequest) { // Serialize the protobuf message to bytes Uint8Array messageBytes = clientRequest.serializeBinary(); - // Send as Uint8Array (which is an ArrayBufferView, compatible with MessageUnion) + // Uint8Array is an ArrayBufferView, which is one of the MessageUnion types + // The unchecked cast is safe because MessageUnion accepts String | ArrayBuffer | ArrayBufferView widget.sendMessage(Js.uncheckedCast(messageBytes), null); } @@ -271,7 +332,11 @@ public String getResourceName() { /** * Responds to this resource request with the given content. * - * @param content the resource content (string or Uint8Array), or null if not found + * @param content the resource content as a String, Uint8Array, or null to indicate + * the resource was not found. If a String is provided, it will be + * UTF-8 encoded before being sent to the server. Uint8Array content + * is sent as-is. + * @throws IllegalArgumentException if content is not a String, Uint8Array, or null */ @JsMethod public void respond(@JsNullable Object content) { From b70f606104a900e7494fa88dce915650f4c359d0 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Mon, 5 Jan 2026 15:13:57 -0600 Subject: [PATCH 47/57] Applying Colin's client proto generation (#DH-20578) --- .../RemoteFileSourceClientRequest.java | 22 +------------------ .../RequestCase.java | 4 ++-- 2 files changed, 3 insertions(+), 23 deletions(-) diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java index 314b642900a..8ba1dcc4846 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java @@ -122,9 +122,6 @@ static RemoteFileSourceClientRequest.ToObjectReturnType create() { @JsProperty RemoteFileSourceClientRequest.ToObjectReturnType.SetExecutionContextFieldType getSetExecutionContext(); - @JsProperty - String getTestCommand(); - @JsProperty void setMetaResponse( RemoteFileSourceClientRequest.ToObjectReturnType.MetaResponseFieldType metaResponse); @@ -135,9 +132,6 @@ void setMetaResponse( @JsProperty void setSetExecutionContext( RemoteFileSourceClientRequest.ToObjectReturnType.SetExecutionContextFieldType setExecutionContext); - - @JsProperty - void setTestCommand(String testCommand); } @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) @@ -245,9 +239,6 @@ static RemoteFileSourceClientRequest.ToObjectReturnType0 create() { @JsProperty RemoteFileSourceClientRequest.ToObjectReturnType0.SetExecutionContextFieldType getSetExecutionContext(); - @JsProperty - String getTestCommand(); - @JsProperty void setMetaResponse( RemoteFileSourceClientRequest.ToObjectReturnType0.MetaResponseFieldType metaResponse); @@ -258,9 +249,6 @@ void setMetaResponse( @JsProperty void setSetExecutionContext( RemoteFileSourceClientRequest.ToObjectReturnType0.SetExecutionContextFieldType setExecutionContext); - - @JsProperty - void setTestCommand(String testCommand); } public static native RemoteFileSourceClientRequest deserializeBinary(Uint8Array bytes); @@ -278,8 +266,6 @@ public static native RemoteFileSourceClientRequest.ToObjectReturnType toObject( public native void clearSetExecutionContext(); - public native void clearTestCommand(); - public native RemoteFileSourceMetaResponse getMetaResponse(); public native int getRequestCase(); @@ -288,14 +274,10 @@ public static native RemoteFileSourceClientRequest.ToObjectReturnType toObject( public native SetExecutionContextRequest getSetExecutionContext(); - public native String getTestCommand(); - public native boolean hasMetaResponse(); public native boolean hasSetExecutionContext(); - public native boolean hasTestCommand(); - public native Uint8Array serializeBinary(); public native void setMetaResponse(); @@ -308,9 +290,7 @@ public static native RemoteFileSourceClientRequest.ToObjectReturnType toObject( public native void setSetExecutionContext(SetExecutionContextRequest value); - public native void setTestCommand(String value); - public native RemoteFileSourceClientRequest.ToObjectReturnType0 toObject(); public native RemoteFileSourceClientRequest.ToObjectReturnType0 toObject(boolean includeInstance); -} +} \ No newline at end of file diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceclientrequest/RequestCase.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceclientrequest/RequestCase.java index 46db67bea6b..a5a156433ed 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceclientrequest/RequestCase.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceclientrequest/RequestCase.java @@ -11,5 +11,5 @@ name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceClientRequest.RequestCase", namespace = JsPackage.GLOBAL) public class RequestCase { - public static int META_RESPONSE, REQUEST_NOT_SET, SET_EXECUTION_CONTEXT, TEST_COMMAND; -} + public static int META_RESPONSE, REQUEST_NOT_SET, SET_EXECUTION_CONTEXT; +} \ No newline at end of file From 52feab62e79df24d17f35b6d559c923feab0fa6e Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Mon, 5 Jan 2026 16:23:49 -0600 Subject: [PATCH 48/57] Changed to runtime dependencies and fixed docs links (#DH-20578) --- server/jetty-app-11/build.gradle | 2 +- server/jetty-app/build.gradle | 2 +- .../client/api/remotefilesource/JsRemoteFileSourceService.java | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/server/jetty-app-11/build.gradle b/server/jetty-app-11/build.gradle index 3a4680c88ca..09baafd32a8 100644 --- a/server/jetty-app-11/build.gradle +++ b/server/jetty-app-11/build.gradle @@ -12,11 +12,11 @@ dependencies { implementation project(':server-jetty-11') implementation project(':extensions-flight-sql') - implementation project(':plugin-remotefilesource') implementation libs.dagger annotationProcessor libs.dagger.compiler + runtimeOnly project(':plugin-remotefilesource') runtimeOnly project(':log-to-slf4j') runtimeOnly project(':logback-print-stream-globals') runtimeOnly project(':logback-logbuffer') diff --git a/server/jetty-app/build.gradle b/server/jetty-app/build.gradle index 728cddc629a..ec03184a602 100644 --- a/server/jetty-app/build.gradle +++ b/server/jetty-app/build.gradle @@ -12,11 +12,11 @@ dependencies { implementation project(':server-jetty') implementation project(':extensions-flight-sql') - implementation project(':plugin-remotefilesource') implementation libs.dagger annotationProcessor libs.dagger.compiler + runtimeOnly project(':plugin-remotefilesource') runtimeOnly project(':log-to-slf4j') runtimeOnly project(':logback-print-stream-globals') runtimeOnly project(':logback-logbuffer') diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index b1070ca3f60..79512bf0f60 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -54,9 +54,11 @@ @JsType(namespace = "dh.remotefilesource", name = "RemoteFileSourceService") public class JsRemoteFileSourceService extends HasEventHandling { /** Event name for generic messages from the server */ + @JsProperty(namespace = "dh.remotefilesource.RemoteFileSourceService") public static final String EVENT_MESSAGE = "message"; /** Event name for resource request events from the server */ + @JsProperty(namespace = "dh.remotefilesource.RemoteFileSourceService") public static final String EVENT_REQUEST_SOURCE = "requestsource"; // Plugin name must match RemoteFileSourcePlugin.name() on the server From f46e97dbfb5bab75b548d1be3e58d145f9112f67 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Tue, 6 Jan 2026 12:47:51 -0600 Subject: [PATCH 49/57] Addressed review comments in RemoteFileSourceClassLoader (#DH-20578) --- .../util/RemoteFileSourceClassLoader.java | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java index 3add4e468cc..470b6c88767 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java @@ -6,6 +6,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.net.MalformedURLException; import java.net.URL; import java.net.URLConnection; import java.net.URLStreamHandler; @@ -78,15 +79,19 @@ public void unregisterProvider(RemoteFileSourceProvider provider) { */ @Override protected URL findResource(String name) { - for (RemoteFileSourceProvider provider : providers) { - if (!provider.isActive() || !provider.canSourceResource(name)) { - continue; + RemoteFileSourceProvider provider = null; + for (RemoteFileSourceProvider candidate : providers) { + if (candidate.isActive() && candidate.canSourceResource(name)) { + provider = candidate; + break; } + } + if (provider != null) { try { return new URL(null, "remotefile://" + name, new RemoteFileURLStreamHandler(provider, name)); - } catch (java.net.MalformedURLException e) { - // Continue to next provider if URL creation fails + } catch (MalformedURLException e) { + // Fall through to parent if URL creation fails } } @@ -169,10 +174,12 @@ public void connect() throws IOException { /** * Returns an input stream that reads from this connection's resource. * - *

    This method ensures the connection is established before returning the stream. + *

    This method calls {@link #connect()} to ensure the connection is established and resource bytes are + * fetched from the provider. The method then verifies that content has been successfully downloaded before + * creating the input stream. * - * @return an input stream that reads from the resource - * @throws IOException if the connection cannot be established or if the resource has no content + * @return an input stream that reads from the fetched resource bytes + * @throws IOException if the connection or content download fails or if the resource has no content */ @Override public InputStream getInputStream() throws IOException { From 5cc36734814d51e4c5e59e36a5fcc0837b7306dd Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Tue, 6 Jan 2026 12:57:44 -0600 Subject: [PATCH 50/57] Addressed review comments in PluginMarker (#DH-20578) --- .../main/java/io/deephaven/plugin/type/PluginMarker.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java b/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java index 158ff2d0a65..2eaa3dcb2f8 100644 --- a/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java +++ b/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java @@ -14,7 +14,8 @@ * in isType(), multiple plugins using PluginMarker would conflict, and whichever is registered * first would intercept all PluginMarker instances. *

    - * This class uses a singleton pattern - one instance per pluginName. + * This class maintains a single instance per pluginName - multiple calls to {@link #forPluginName(String)} + * with the same name will return the same instance. */ public class PluginMarker { private static final Map INSTANCES = new ConcurrentHashMap<>(); @@ -22,7 +23,7 @@ public class PluginMarker { private final String pluginName; /** - * Private constructor - use forPluginName() to get singleton instances. + * Private constructor - use forPluginName() to get instances. * * @param pluginName the plugin name identifier (should match the plugin's name() method) */ @@ -31,10 +32,10 @@ private PluginMarker(String pluginName) { } /** - * Gets the singleton PluginMarker instance for the specified plugin name. + * Gets the PluginMarker instance for the specified plugin name, creating it if necessary. * * @param pluginName the plugin name identifier (should match the plugin's name() method) - * @return the singleton PluginMarker for this plugin name + * @return the PluginMarker instance for this plugin name * @throws IllegalArgumentException if pluginName is null or empty */ public static PluginMarker forPluginName(String pluginName) { From 7ad0a36e52b87e942f61bb42df8dba7fcb697fbc Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Tue, 6 Jan 2026 13:08:16 -0600 Subject: [PATCH 51/57] Addressed review comments in RemoteFileSourceMessageStream (#DH-20578) --- .../RemoteFileSourceMessageStream.java | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java index 9cc3879b80f..cbafb2e1dc4 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java @@ -17,8 +17,6 @@ import io.deephaven.proto.backplane.grpc.SetExecutionContextResponse; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.UUID; @@ -88,7 +86,7 @@ public boolean canSourceResource(String resourceName) { for (String contextResourcePath : resourcePaths) { if (resourceName.equals(contextResourcePath)) { - log.info().append("Can source: ").append(resourceName).endl(); + log.debug().append("Can source: ").append(resourceName).endl(); return true; } } @@ -187,15 +185,6 @@ public static void clearExecutionContext() { } } - /** - * Gets the current execution context. - * - * @return the execution context - */ - public static RemoteFileSourceExecutionContext getExecutionContext() { - return executionContext; - } - /** * Handles incoming data from the client. * Parses RemoteFileSourceClientRequest messages and processes meta responses @@ -353,7 +342,7 @@ public static class RemoteFileSourceExecutionContext { public RemoteFileSourceExecutionContext(RemoteFileSourceMessageStream activeMessageStream, List resourcePaths) { this.activeMessageStream = activeMessageStream; - this.resourcePaths = resourcePaths != null ? resourcePaths : Collections.emptyList(); + this.resourcePaths = resourcePaths; } /** @@ -368,10 +357,10 @@ public RemoteFileSourceMessageStream getActiveMessageStream() { /** * Gets the resource paths that should be resolved from the remote source. * - * @return a copy of the list of resource paths + * @return the list of resource paths */ public List getResourcePaths() { - return new ArrayList<>(resourcePaths); + return resourcePaths; } } } From 325f68893e968d24781cc0b1fcd44ee8ff212016 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Tue, 6 Jan 2026 15:33:28 -0600 Subject: [PATCH 52/57] Addressed review comments in JsRemoteFileSourceService (#DH-20578) --- .../JsRemoteFileSourceService.java | 84 +++++++------------ .../ResourceContentUnion.java | 49 +++++++++++ 2 files changed, 81 insertions(+), 52 deletions(-) create mode 100644 web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/ResourceContentUnion.java diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index 79512bf0f60..957ee628c4b 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -5,7 +5,6 @@ import com.vertispan.tsdefs.annotations.TsInterface; import com.vertispan.tsdefs.annotations.TsName; -import com.vertispan.tsdefs.annotations.TsTypeRef; import elemental2.core.Uint8Array; import elemental2.dom.DomGlobal; import elemental2.dom.TextEncoder; @@ -28,6 +27,7 @@ import io.deephaven.web.client.api.event.HasEventHandling; import io.deephaven.web.client.api.widget.JsWidget; import io.deephaven.web.client.api.widget.WidgetMessageDetails; +import io.deephaven.web.client.fu.LazyPromise; import jsinterop.annotations.JsIgnore; import jsinterop.annotations.JsMethod; import jsinterop.annotations.JsNullable; @@ -70,10 +70,9 @@ public class JsRemoteFileSourceService extends HasEventHandling { private final JsWidget widget; // Track pending setExecutionContext requests - private final Map> pendingSetExecutionContextRequests = new HashMap<>(); + private final Map> pendingSetExecutionContextRequests = new HashMap<>(); private int requestIdCounter = 0; - @JsIgnore private JsRemoteFileSourceService(JsWidget widget) { this.widget = widget; } @@ -84,7 +83,6 @@ private JsRemoteFileSourceService(JsWidget widget) { * @param connection the worker connection to use * @return a promise that resolves to the FlightInfo for the plugin fetch */ - @JsIgnore private static Promise fetchPluginFlightInfo(WorkerConnection connection) { // Create a new export ticket for the result Ticket resultTicket = connection.getTickets().newExportTicket(); @@ -119,7 +117,7 @@ private static Promise fetchPluginFlightInfo(WorkerConnection connec * @return a promise that resolves to a RemoteFileSourceService instance with an active message stream */ @JsIgnore - public static Promise fetchPlugin(@TsTypeRef(Object.class) WorkerConnection connection) { + public static Promise fetchPlugin(WorkerConnection connection) { return fetchPluginFlightInfo(connection) .then(flightInfo -> { // The first endpoint contains the ticket for the plugin instance. @@ -156,7 +154,6 @@ public static Promise fetchPlugin(@TsTypeRef(Object.c * * @return a promise that resolves to this service instance when the connection is established */ - @JsIgnore private Promise connect() { widget.addEventListener("message", this::handleMessage); return widget.refetch().then(w -> Promise.resolve(this)); @@ -167,7 +164,6 @@ private Promise connect() { * * @param event the message event from the server */ - @JsIgnore private void handleMessage(Event event) { Uint8Array payload = event.getDetail().getDataAsU8(); @@ -195,7 +191,6 @@ private void handleMessage(Event event) { * * @param message the server request message */ - @JsIgnore private void handleMetaRequest(RemoteFileSourceServerRequest message) { RemoteFileSourceMetaRequest request = message.getMetaRequest(); DomGlobal.setTimeout(ignore -> fireEvent(EVENT_REQUEST_SOURCE, @@ -207,14 +202,12 @@ private void handleMetaRequest(RemoteFileSourceServerRequest message) { * * @param message the server request message */ - @JsIgnore private void handleSetExecutionContextResponse(RemoteFileSourceServerRequest message) { String requestId = message.getRequestId(); - Promise.PromiseExecutorCallbackFn.ResolveCallbackFn resolveCallback = - pendingSetExecutionContextRequests.remove(requestId); - if (resolveCallback != null) { + LazyPromise promise = pendingSetExecutionContextRequests.remove(requestId); + if (promise != null) { SetExecutionContextResponse response = message.getSetExecutionContextResponse(); - resolveCallback.onInvoke(response.getSuccess()); + promise.succeed(response.getSuccess()); } } @@ -223,7 +216,6 @@ private void handleSetExecutionContextResponse(RemoteFileSourceServerRequest mes * * @param event the message event */ - @JsIgnore private void handleUnknownMessage(Event event) { DomGlobal.setTimeout(ignore -> fireEvent(EVENT_MESSAGE, event.getDetail()), 0); } @@ -239,27 +231,19 @@ private void handleUnknownMessage(Event event) { */ @JsMethod public Promise setExecutionContext(@JsOptional String[] resourcePaths) { - return new Promise<>((resolve, reject) -> { - // Generate a unique request ID - String requestId = "setExecutionContext-" + (requestIdCounter++); - - // Store the resolve callback to call when we get the acknowledgment - pendingSetExecutionContextRequests.put(requestId, resolve); - - // Set a timeout to reject the promise if no response is received - DomGlobal.setTimeout(ignore -> { - Promise.PromiseExecutorCallbackFn.ResolveCallbackFn callback = - pendingSetExecutionContextRequests.remove(requestId); - if (callback != null) { - // Request timed out - reject the promise - reject.onInvoke("setExecutionContext request timed out after " - + SET_EXECUTION_CONTEXT_TIMEOUT_MS + "ms"); - } - }, SET_EXECUTION_CONTEXT_TIMEOUT_MS); + // Generate a unique request ID + String requestId = "setExecutionContext-" + (requestIdCounter++); - RemoteFileSourceClientRequest clientRequest = getSetExecutionContextRequest(resourcePaths, requestId); - sendClientRequest(clientRequest); - }); + // Create a lazy promise that will be resolved when we get the response + LazyPromise promise = new LazyPromise<>(); + pendingSetExecutionContextRequests.put(requestId, promise); + + // Send the request + RemoteFileSourceClientRequest clientRequest = getSetExecutionContextRequest(resourcePaths, requestId); + sendClientRequest(clientRequest); + + // Return a promise with built-in timeout + return promise.asPromise(SET_EXECUTION_CONTEXT_TIMEOUT_MS); } /** @@ -273,9 +257,7 @@ public Promise setExecutionContext(@JsOptional String[] resourcePaths) SetExecutionContextRequest setContextRequest = new SetExecutionContextRequest(); if (resourcePaths != null) { - for (String resourcePath : resourcePaths) { - setContextRequest.addResourcePaths(resourcePath); - } + setContextRequest.setResourcePathsList(resourcePaths); } RemoteFileSourceClientRequest clientRequest = new RemoteFileSourceClientRequest(); @@ -289,7 +271,6 @@ public Promise setExecutionContext(@JsOptional String[] resourcePaths) * * @param clientRequest the client request to send */ - @JsIgnore private void sendClientRequest(RemoteFileSourceClientRequest clientRequest) { // Serialize the protobuf message to bytes Uint8Array messageBytes = clientRequest.serializeBinary(); @@ -302,7 +283,6 @@ private void sendClientRequest(RemoteFileSourceClientRequest clientRequest) { /** * Closes the message stream connection to the server. */ - @JsMethod public void close() { widget.close(); } @@ -312,12 +292,11 @@ public void close() { * respond() method. */ @TsInterface - @TsName(namespace = "dh.remotefilesource", name = "ResourceRequestEvent") + @TsName(namespace = "dh.remotefilesource") public class ResourceRequestEvent { private final String requestId; private final RemoteFileSourceMetaRequest protoRequest; - @JsIgnore public ResourceRequestEvent(String requestId, RemoteFileSourceMetaRequest protoRequest) { this.requestId = requestId; this.protoRequest = protoRequest; @@ -334,14 +313,15 @@ public String getResourceName() { /** * Responds to this resource request with the given content. * - * @param content the resource content as a String, Uint8Array, or null to indicate - * the resource was not found. If a String is provided, it will be - * UTF-8 encoded before being sent to the server. Uint8Array content - * is sent as-is. - * @throws IllegalArgumentException if content is not a String, Uint8Array, or null + * @param content the resource content (String | Uint8Array | null): + *

      + *
    • String - will be UTF-8 encoded before sending to server
      • + *
    • Uint8Array - sent as-is to server
      • + *
    • null - indicates resource was not found
      • + *
    */ @JsMethod - public void respond(@JsNullable Object content) { + public void respond(@JsNullable ResourceContentUnion content) { // Build RemoteFileSourceMetaResponse proto RemoteFileSourceMetaResponse response = new RemoteFileSourceMetaResponse(); @@ -352,12 +332,12 @@ public void respond(@JsNullable Object content) { } else { response.setFound(true); - // Convert content to bytes - if (content instanceof String) { + // Convert content to bytes using union type methods + if (content.isString()) { TextEncoder textEncoder = new TextEncoder(); - response.setContent(textEncoder.encode((String) content)); - } else if (content instanceof Uint8Array) { - response.setContent((Uint8Array) content); + response.setContent(textEncoder.encode(content.asString())); + } else if (content.isUint8Array()) { + response.setContent(content.asUint8Array()); } else { throw new IllegalArgumentException("Content must be a String, Uint8Array, or null"); } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/ResourceContentUnion.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/ResourceContentUnion.java new file mode 100644 index 00000000000..26135d2f1ba --- /dev/null +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/ResourceContentUnion.java @@ -0,0 +1,49 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.web.client.api.remotefilesource; + +import com.vertispan.tsdefs.annotations.TsUnion; +import com.vertispan.tsdefs.annotations.TsUnionMember; +import elemental2.core.Uint8Array; +import jsinterop.annotations.JsOverlay; +import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsType; +import jsinterop.base.Js; + +/** + * Union type for resource content that can be either a String or Uint8Array. + */ +@TsUnion +@JsType(name = "?", namespace = JsPackage.GLOBAL, isNative = true) +public interface ResourceContentUnion { + @JsOverlay + static ResourceContentUnion of(Object o) { + return Js.cast(o); + } + + @JsOverlay + default boolean isString() { + // Cast to (Object) since Java only "knows" that `this` is `ResourceContentUnion` type which cannot have a + // subclass that is also a String. + return (Object) this instanceof String; + } + + @JsOverlay + default boolean isUint8Array() { + return this instanceof Uint8Array; + } + + @TsUnionMember + @JsOverlay + default String asString() { + return Js.cast(this); + } + + @TsUnionMember + @JsOverlay + default Uint8Array asUint8Array() { + return (Uint8Array) this; + } +} + From 1cb84d4e48dbbce803eb839a85004727d37acd5e Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Tue, 13 Jan 2026 17:13:45 -0600 Subject: [PATCH 53/57] Changed to getResource (#DH-20578) --- .../deephaven/engine/util/RemoteFileSourceClassLoader.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java index 470b6c88767..c4b3601a919 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java @@ -68,7 +68,7 @@ public void unregisterProvider(RemoteFileSourceProvider provider) { } /** - * Finds the resource with the specified name by checking registered providers. + * Gets the resource with the specified name by checking registered providers. * *

    This method iterates through all registered providers to see if any can source the requested resource. * If a provider can handle the resource, a custom URL with protocol "remotefile://" is returned. @@ -78,7 +78,7 @@ public void unregisterProvider(RemoteFileSourceProvider provider) { * @return a URL for reading the resource, or null if the resource could not be found */ @Override - protected URL findResource(String name) { + public URL getResource(String name) { RemoteFileSourceProvider provider = null; for (RemoteFileSourceProvider candidate : providers) { if (candidate.isActive() && candidate.canSourceResource(name)) { @@ -95,7 +95,7 @@ protected URL findResource(String name) { } } - return super.findResource(name); + return super.getResource(name); } /** From 71c58b89f75bb779b48741447452568737e69902 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Tue, 13 Jan 2026 18:12:06 -0600 Subject: [PATCH 54/57] Addressed review comments (#DH-20578) --- .../util/RemoteFileSourceClassLoader.java | 25 +++++++++++++++++-- .../engine/util/GroovyDeephavenSession.java | 2 +- .../RemoteFileSourceMessageStream.java | 16 +++++++++--- .../JsRemoteFileSourceService.java | 22 +++------------- 4 files changed, 41 insertions(+), 24 deletions(-) diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java index c4b3601a919..9a28f6323c0 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java @@ -35,9 +35,27 @@ public class RemoteFileSourceClassLoader extends ClassLoader { * * @param parent the parent class loader for delegation */ - public RemoteFileSourceClassLoader(ClassLoader parent) { + private RemoteFileSourceClassLoader(ClassLoader parent) { super(parent); - instance = this; + } + + /** + * Initializes the singleton RemoteFileSourceClassLoader instance with the specified parent class loader. + * + *

    This method must be called exactly once before any calls to {@link #getInstance()}. The method is + * synchronized to prevent race conditions when multiple threads attempt initialization. + * + * @param parent the parent class loader for delegation + * @return the newly created singleton instance + * @throws IllegalStateException if the instance has already been initialized + */ + public static synchronized RemoteFileSourceClassLoader initialize(ClassLoader parent) { + if (instance != null) { + throw new IllegalStateException("RemoteFileSourceClassLoader is already initialized"); + } + + instance = new RemoteFileSourceClassLoader(parent); + return instance; } /** @@ -46,6 +64,9 @@ public RemoteFileSourceClassLoader(ClassLoader parent) { * @return the singleton instance, or null if not yet initialized */ public static RemoteFileSourceClassLoader getInstance() { + if (instance == null) { + throw new IllegalStateException("RemoteFileSourceClassLoader is not yet initialized"); + } return instance; } diff --git a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java index 91dcdf88187..fa098c1e87b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java @@ -96,7 +96,7 @@ public class GroovyDeephavenSession extends AbstractScriptSession mapping = new ConcurrentHashMap<>(); @Override diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java index cbafb2e1dc4..a50316aa75c 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java @@ -159,10 +159,20 @@ public boolean isActive() { /** * Sets the execution context with the active message stream and resource paths. - * This should be called when a script execution begins. * - * @param messageStream the message stream to set as active - * @param resourcePaths list of resource paths to resolve from remote source + *

    This static method establishes which message stream instance should be considered "active" for + * resource requests, and which resource paths should be resolved from that remote source. Only one + * execution context can be active at a time across all instances. + * + *

    In multi-client scenarios (Community Core), this ensures that only the + * message stream for the currently executing script is active, preventing resource requests from + * being serviced by the wrong client connection. + * + *

    Typical Usage: Called at the beginning of script execution to establish which .groovy + * files should be sourced from the remote client rather than the local classpath. + * + * @param messageStream the message stream to set as active (must not be null) + * @param resourcePaths list of resource paths (e.g., "package/MyScript.groovy") to resolve from remote source * @throws IllegalArgumentException if messageStream is null */ public static void setExecutionContext(RemoteFileSourceMessageStream messageStream, List resourcePaths) { diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index 957ee628c4b..3de111d1574 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -47,16 +47,11 @@ *

    * Events: *

      - *
    • {@link #EVENT_MESSAGE}: Fired for unrecognized messages from the server
      • *
    • {@link #EVENT_REQUEST_SOURCE}: Fired when the server requests a resource from the client
      • *
    */ @JsType(namespace = "dh.remotefilesource", name = "RemoteFileSourceService") public class JsRemoteFileSourceService extends HasEventHandling { - /** Event name for generic messages from the server */ - @JsProperty(namespace = "dh.remotefilesource.RemoteFileSourceService") - public static final String EVENT_MESSAGE = "message"; - /** Event name for resource request events from the server */ @JsProperty(namespace = "dh.remotefilesource.RemoteFileSourceService") public static final String EVENT_REQUEST_SOURCE = "requestsource"; @@ -155,7 +150,7 @@ public static Promise fetchPlugin(WorkerConnection co * @return a promise that resolves to this service instance when the connection is established */ private Promise connect() { - widget.addEventListener("message", this::handleMessage); + widget.addEventListener(JsWidget.EVENT_MESSAGE, this::handleMessage); return widget.refetch().then(w -> Promise.resolve(this)); } @@ -171,9 +166,8 @@ private void handleMessage(Event event) { try { message = RemoteFileSourceServerRequest.deserializeBinary(payload); } catch (Exception e) { - // Failed to parse as proto, fire generic message event - handleUnknownMessage(event); - return; + // Failed to parse as proto + throw new IllegalStateException("Received unparseable message from server", e); } // Route the parsed message to the appropriate handler @@ -182,7 +176,7 @@ private void handleMessage(Event event) { } else if (message.hasSetExecutionContextResponse()) { handleSetExecutionContextResponse(message); } else { - handleUnknownMessage(event); + throw new IllegalStateException("Received unknown message type from server"); } } @@ -211,14 +205,6 @@ private void handleSetExecutionContextResponse(RemoteFileSourceServerRequest mes } } - /** - * Handles an unknown or unparseable message from the server. - * - * @param event the message event - */ - private void handleUnknownMessage(Event event) { - DomGlobal.setTimeout(ignore -> fireEvent(EVENT_MESSAGE, event.getDetail()), 0); - } /** * Sets the execution context on the server to identify this message stream as active From 1f775ccbccce9a3889c956bbbdbd80b83f19630b Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Tue, 13 Jan 2026 18:18:44 -0600 Subject: [PATCH 55/57] Addressed review comments (#DH-20578) --- .../util/RemoteFileSourceClassLoader.java | 5 ++++- .../RemoteFileSourceMessageStream.java | 19 ++++++------------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java index 9a28f6323c0..7e751a45998 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java @@ -61,7 +61,10 @@ public static synchronized RemoteFileSourceClassLoader initialize(ClassLoader pa /** * Returns the singleton instance of the RemoteFileSourceClassLoader. * - * @return the singleton instance, or null if not yet initialized + *

    This method requires that {@link #initialize(ClassLoader)} has been called first. + * + * @return the singleton instance + * @throws IllegalStateException if the instance has not yet been initialized via {@link #initialize(ClassLoader)} */ public static RemoteFileSourceClassLoader getInstance() { if (instance == null) { diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java index a50316aa75c..f09bf2a4c1b 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java @@ -216,7 +216,8 @@ public void onData(ByteBuffer payload, Object... references) throws ObjectCommun } else if (message.hasSetExecutionContext()) { handleSetExecutionContext(message.getRequestId(), message.getSetExecutionContext().getResourcePathsList()); } else { - log.warn().append("Received unknown message type from client").endl(); + log.error().append("Received unknown message type from client").endl(); + throw new ObjectCommunicationException("Received unknown message type from client"); } } catch (InvalidProtocolBufferException e) { log.error().append("Failed to parse RemoteFileSourceClientRequest: ").append(e).endl(); @@ -311,13 +312,8 @@ public void onClose() { */ private void registerWithClassLoader() { RemoteFileSourceClassLoader classLoader = RemoteFileSourceClassLoader.getInstance(); - - if (classLoader != null) { - classLoader.registerProvider(this); - log.info().append("Registered RemoteFileSourceMessageStream provider with RemoteFileSourceClassLoader").endl(); - } else { - log.warn().append("RemoteFileSourceClassLoader not available").endl(); - } + classLoader.registerProvider(this); + log.info().append("Registered RemoteFileSourceMessageStream provider with RemoteFileSourceClassLoader").endl(); } /** @@ -325,11 +321,8 @@ private void registerWithClassLoader() { */ private void unregisterFromClassLoader() { RemoteFileSourceClassLoader classLoader = RemoteFileSourceClassLoader.getInstance(); - - if (classLoader != null) { - classLoader.unregisterProvider(this); - log.info().append("Unregistered RemoteFileSourceMessageStream provider from RemoteFileSourceClassLoader").endl(); - } + classLoader.unregisterProvider(this); + log.info().append("Unregistered RemoteFileSourceMessageStream provider from RemoteFileSourceClassLoader").endl(); } From d5ba658f3ec251f5d62774b96dea312e8a51510a Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 14 Jan 2026 12:17:32 -0600 Subject: [PATCH 56/57] Addressed review comments (#DH-20578) --- .../deephaven/web/client/api/CoreClient.java | 7 ++++- .../JsRemoteFileSourceService.java | 29 ++++++++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java b/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java index 88da02de07c..e69e0f01814 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java @@ -47,6 +47,7 @@ public class CoreClient extends HasEventHandling { LOGIN_TYPE_ANONYMOUS = "anonymous"; private final IdeConnection ideConnection; + private Promise remoteFileSourceServicePromise; public CoreClient(String serverUrl, @TsTypeRef(ConnectOptions.class) @JsOptional Object connectOptions) { ideConnection = new IdeConnection(serverUrl, connectOptions); @@ -143,7 +144,11 @@ public Promise onConnected(@JsOptional Double timeoutInMillis) { } public Promise getRemoteFileSourceService() { - return JsRemoteFileSourceService.fetchPlugin(ideConnection.connection.get()); + if (remoteFileSourceServicePromise == null) { + remoteFileSourceServicePromise = JsRemoteFileSourceService.fetchPlugin(ideConnection.connection.get()); + } + + return remoteFileSourceServicePromise; } public Promise getServerConfigValues() { diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index 3de111d1574..5ca57e60266 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -23,9 +23,11 @@ import io.deephaven.web.client.api.Callbacks; import io.deephaven.web.client.api.JsProtobufUtils; import io.deephaven.web.client.api.event.Event; +import io.deephaven.web.client.api.event.EventFn; import io.deephaven.web.client.api.WorkerConnection; import io.deephaven.web.client.api.event.HasEventHandling; import io.deephaven.web.client.api.widget.JsWidget; +import io.deephaven.web.shared.fu.RemoverFn; import io.deephaven.web.client.api.widget.WidgetMessageDetails; import io.deephaven.web.client.fu.LazyPromise; import jsinterop.annotations.JsIgnore; @@ -47,7 +49,10 @@ *

    * Events: *

      - *
    • {@link #EVENT_REQUEST_SOURCE}: Fired when the server requests a resource from the client
      • + *
    • {@link #EVENT_REQUEST_SOURCE}: Fired when the server requests a resource from the client. + * This event MUST have exactly one listener registered. Attempting to register more than one listener + * will throw an IllegalStateException. Receiving a resource request without a registered listener + * will also throw an IllegalStateException.
      • *
    */ @JsType(namespace = "dh.remotefilesource", name = "RemoteFileSourceService") @@ -72,6 +77,23 @@ private JsRemoteFileSourceService(JsWidget widget) { this.widget = widget; } + /** + * Overrides addEventListener to enforce that EVENT_REQUEST_SOURCE can only have one listener. + * + * @param name the name of the event to listen for + * @param callback a function to call when the event occurs + * @return Returns a cleanup function. + * @param the type of the data that the event will provide + */ + @Override + public RemoverFn addEventListener(String name, EventFn callback) { + if (EVENT_REQUEST_SOURCE.equals(name) && hasListeners(EVENT_REQUEST_SOURCE)) { + throw new IllegalStateException( + "EVENT_REQUEST_SOURCE already has a listener. Only one listener is allowed for this event."); + } + return super.addEventListener(name, callback); + } + /** * Fetches the FlightInfo for the plugin fetch command. * @@ -186,6 +208,11 @@ private void handleMessage(Event event) { * @param message the server request message */ private void handleMetaRequest(RemoteFileSourceServerRequest message) { + if (!hasListeners(EVENT_REQUEST_SOURCE)) { + throw new IllegalStateException( + "Received resource request from server but no listener is registered for EVENT_REQUEST_SOURCE. " + + "A listener must be registered to handle resource requests."); + } RemoteFileSourceMetaRequest request = message.getMetaRequest(); DomGlobal.setTimeout(ignore -> fireEvent(EVENT_REQUEST_SOURCE, new ResourceRequestEvent(message.getRequestId(), request)), 0); From 1cdbac4ae146f0373ef2306f526bb940196fb8ad Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 14 Jan 2026 12:28:50 -0600 Subject: [PATCH 57/57] Removed timeout (#DH-20578) --- .../client/api/remotefilesource/JsRemoteFileSourceService.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index 5ca57e60266..b706d895467 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -214,8 +214,7 @@ private void handleMetaRequest(RemoteFileSourceServerRequest message) { + "A listener must be registered to handle resource requests."); } RemoteFileSourceMetaRequest request = message.getMetaRequest(); - DomGlobal.setTimeout(ignore -> fireEvent(EVENT_REQUEST_SOURCE, - new ResourceRequestEvent(message.getRequestId(), request)), 0); + fireEvent(EVENT_REQUEST_SOURCE, new ResourceRequestEvent(message.getRequestId(), request)); } /**