feat: DH-19901: Use grpc-java in JS API (#7878)

This is in part a backport from DHE, bringing grpc-java and
protobuf-java support to the JS API. Removes the need to load
dh-internal.js by compiling the Java protos and grpc services to JS,
allowing easier code reuse and sharing, supporting more gRPC features,
and removing the need to work around limitations around longs, headers,
etc. Some of these changes will allow us to simplify DHE to better share
code after this merges and is released.

Some grpc-java semantics are not exactly the same as in a JVM -
Context's threadlocals won't play nicely with browser Promises for
example, so must be explicitly propagated when used.

More cleanup is possible than has been done here - BiDiStream should no
longer be necessary, but can just be a normal pair of StreamObservers.
All of web/client-backplane can be deleted, but that would substantially
bloat this patch, so I've left it out for now.

Fixes DH-17942
Fixes DH-21675

Author: niloc132

buildSrc/src/main/groovy/GwtTools.groovy

@@ -59,6 +59,11 @@ class GwtTools {
                 checkAssertions = true
                 setExtraArgs('-includeJsInteropExports', 'io.deephaven.*')
                 if (gwtDev) {
+                    extraArgs = [
+                            '-includeJsInteropExports', 'io.deephaven.*',
+                            '-setProperty', 'gwt.logging.logLevel=FINE',
+                            '-setProperty', 'jre.logging.logLevel=ALL',
+                    ]
                     saveSource = true
                     extra = extras
                     logLevel = 'INFO'
@@ -80,6 +85,8 @@ class GwtTools {
         def libs = p.getExtensions().getByType(VersionCatalogsExtension).named("libs")
         def gwtVersion = libs.findVersion("gwt").map(VersionConstraint::getRequiredVersion).orElseThrow()
         def gwtJettyVersion = libs.findVersion("gwtJetty").map(VersionConstraint::getRequiredVersion).orElseThrow()
+        def protobufVers = libs.findVersion('protobuf-gwt').get().requiredVersion
+        def grpcVers = libs.findVersion('grpc-gwt').get().requiredVersion
 
         gwt.gwtVersion = gwtVersion
         gwt.jettyVersion = gwtJettyVersion
@@ -91,7 +98,22 @@ class GwtTools {
                         .using(sub.module("org.gwtproject:gwt-user:${gwtVersion}"))
                 sub.substitute(sub.module("com.google.gwt:gwt-dev"))
                         .using(sub.module("org.gwtproject:gwt-dev:${gwtVersion}"))
+                sub.substitute(sub.module('com.google.protobuf:protobuf-java'))
+                        .using(sub.module("com.vertispan.protobuf:protobuf-gwt:${protobufVers}"))
+                sub.substitute(sub.module('io.grpc:grpc-api'))
+                        .using(sub.module("com.vertispan.grpc:grpc-gwt:${grpcVers}"))
+                sub.substitute(sub.module('io.grpc:grpc-core'))
+                        .using(sub.module("com.vertispan.grpc:grpc-gwt:${grpcVers}"))
+                sub.substitute(sub.module('io.grpc:grpc-stub'))
+                        .using(sub.module("com.vertispan.grpc:grpc-gwt:${grpcVers}"))
+                sub.substitute(sub.module('io.grpc:grpc-protobuf'))
+                        .using(sub.module("com.vertispan.grpc:grpc-gwt:${grpcVers}"))
+                sub.substitute(sub.module('io.grpc:grpc-protobuf-stub'))
+                        .using(sub.module("com.vertispan.grpc:grpc-gwt:${grpcVers}"))
             }
+
+            c.exclude(group: 'io.grpc', module: 'grpc-netty')
+            c.exclude(group: 'com.google.protobuf', module: 'protobuf-java-util')
         }
         String warPath = new File(p.buildDir, 'gwt').absolutePath
 
@@ -114,7 +136,7 @@ class GwtTools {
             /** Fail compilation if any input file contains an error. */
             strict = true
             /** Specifies Java source level. ("1.6", "1.7")*/
-            sourceLevel = "11"
+            sourceLevel = "17"
             /** The number of local workers to use when compiling permutations. */
             localWorkers = 1
             /** Emit extra information allow chrome dev tools to display Java identifiers in many places instead of JavaScript functions. (NONE, ONLY_METHOD_NAME, ABBREVIATED, FULL)*/

extensions/protobuf/build.gradle

@@ -46,6 +46,6 @@ spotless {
 
 protobuf {
     protoc {
-        artifact = "com.google.protobuf:protoc:${libs.versions.protobuf.get()}"
+        artifact = libs.protoc.get().toString()
     }
 }

gradle/libs.versions.toml

@@ -27,7 +27,7 @@ deephaven-hash = "0.5.0"
 deephaven-suan-shu = "0.1.1"
 dev-dirs = "26"
 dsi = "8.5.18"
-elemental = "1.2.3"
+elemental = "1.3.2"
 f4b6a3 = "6.1.1"
 flatbuffers = "25.2.10"
 freemarker = "2.3.34"
@@ -42,7 +42,12 @@ groovy = "3.0.25"
 grpc = "1.76.2"
 netty = "4.1.132.Final"
 
-guava = "33.4.8-jre"
+protobuf-gwt='4.33.2-2'
+grpc-gwt='1.76.2-2'
+
+grpc-web-gwt-fetch='1.2.0'
+
+guava = "33.5.0-jre"
 gwt = "2.12.2"
 # used by GwtTools
 gwtJetty = "9.4.44.v20210927"
@@ -63,7 +68,7 @@ jetbrains = "26.1.0"
 jetty = "12.0.33"
 jetty11 = "11.0.20"
 jpy = "1.3.0"
-jsinterop = "2.0.2"
+jsinterop = "2.1.0"
 # google is annoying, and have different versions released for the same groupId
 jsinterop-base = "1.1.1"
 logback = "1.5.32"
@@ -203,7 +208,12 @@ netty-bom = { module = "io.netty:netty-bom", version.ref = "netty" }
 
 google-rpc-protos = { module="com.google.api.grpc:proto-google-common-protos", version.ref="google-rpc-protos" }
 
+protobuf-gwt = { module = 'com.vertispan.protobuf:protobuf-gwt', version.ref = 'protobuf-gwt' }
+grpc-gwt = { module = 'com.vertispan.grpc:grpc-gwt', version.ref = 'grpc-gwt' }
+grpc-gwt-fetch = { module = 'com.vertispan.grpc:grpc-web-gwt-fetch', version.ref = 'grpc-web-gwt-fetch' }
+
 guava = { module = "com.google.guava:guava", version.ref = "guava" }
+guava-gwt = { module = "com.google.guava:guava-gwt", version.ref = "guava" }
 
 gwt-user = { module = "org.gwtproject:gwt-user", version.ref = "gwt" }
 
@@ -300,6 +310,7 @@ postgresql = { module = "org.postgresql:postgresql", version.ref = "postgresql"
 
 protobuf-java = { module = "com.google.protobuf:protobuf-java", version.ref = "protobuf" }
 protobuf-java-util = { module = "com.google.protobuf:protobuf-java-util", version.ref = "protobuf" }
+protoc = { module = "com.google.protobuf:protoc", version.ref = "protobuf" }
 
 randelshofer-fastdoubleparser = { module = "ch.randelshofer:fastdoubleparser", version.ref = "randelshofer" }
 

grpc-java/grpc-servlet-websocket-jakarta/src/main/java/io/grpc/servlet/web/websocket/MultiplexedWebSocketServerStream.java

@@ -220,7 +220,9 @@ public void onMessage(ByteBuffer message) throws IOException {
         }
 
         // Having already stripped the control flow byte, the rest of the payload is our request message
-        stream.inboundDataReceived(ReadableBuffers.wrap(message), false);
+        ByteBuffer copy = ByteBuffer.allocate(message.remaining());
+        copy.put(message);
+        stream.inboundDataReceived(ReadableBuffers.wrap(copy.flip()), false);
     }
 
     @Override

proto/proto-backplane-grpc/src/main/resources/io/deephaven/flightjs/protocol/BrowserFlight.gwt.xml

@@ -0,0 +1,3 @@
+<module>
+    <source path="" />
+</module>

proto/proto-backplane-grpc/src/main/resources/io/deephaven/proto/backplane/DeephavenCoreProto.gwt.xml

@@ -0,0 +1,5 @@
+<module>
+    <inherits name="io.grpc.Grpc" />
+    <source path="script" />
+    <source path="grpc" />
+</module>

web/client-api/client-api.gradle

@@ -39,8 +39,10 @@ configurations {
 
 dependencies {
     gwt libs.gwt.user
-    gwt platform(libs.grpc.bom)
-    gwt libs.grpc.api
+    api libs.protobuf.gwt
+    api libs.grpc.gwt
+    api libs.grpc.gwt.fetch
+    api libs.guava.gwt
 
     gwt project(':engine-chunk')
     gwt project(':extensions-barrage')
@@ -63,9 +65,12 @@ dependencies {
     gwt libs.vertispan.nio.gwt
     gwt libs.vertispan.flatbuffers.gwt
 
-    js project(path: ':proto:raw-js-openapi', configuration: 'js')
+    gwt project(':proto:proto-backplane-grpc')
+
+    gwt libs.arrow.flight.core
 
     gwt libs.guava
+    gwt libs.guava.gwt
 
     gwt libs.elemental.core
     gwt libs.elemental.promise

web/client-api/gradle.properties

@@ -1 +1,3 @@
 io.deephaven.project.ProjectType=JAVA_PUBLIC
+languageLevel=17
+testLanguageLevel=17

web/client-api/src/main/java/io/deephaven/web/DeephavenApi.gwt.xml

@@ -4,22 +4,27 @@
   <inherits name="elemental2.dom.Dom"/>
   <inherits name="com.google.flatbuffers.FlatBuffers"/>
   <inherits name="org.apache.arrow.flatbuf.FlightFlatbufFormat"/>
+  <inherits name="org.apache.arrow.flight.impl.FlightProto"/>
   <inherits name="io.deephaven.barrage.flatbuf.BarrageFlatbufFormat"/>
 
   <inherits name="io.deephaven.base.Base" />
   <inherits name="io.deephaven.Util" />
   <inherits name="io.deephaven.io.IO" />
   <inherits name="io.deephaven.chunk.Chunk" />
   <inherits name="io.deephaven.extensions.barrage.Barrage" />
+  <inherits name="io.deephaven.flightjs.protocol.BrowserFlight" />
+  <inherits name="io.deephaven.proto.backplane.DeephavenCoreProto" />
+
+  <inherits name="com.vertispan.grpc.fetch.Fetch" />
+
+  <inherits name="com.google.gwt.logging.Logging"/>
 
   <define-property name="dh.dev" values="true, false" />
   <source path="client" />
   <source path="shared" />
   <super-source path="super" />
   <set-property name="dh.dev" value="false" />
 
-  <inherits name="io.deephaven.javascript.proto.dhinternal.DeephavenInternal" />
-
   <inherits name="com.google.gwt.i18n.I18N" />
   <inherits name="io.deephaven.web.DeephavenSharedApi" />
 
@@ -29,6 +34,13 @@
   <add-linker name="dh_linker" />
 
   <set-configuration-property name="user.agent.runtimeWarning" value="false"/>
+  <set-configuration-property name="document.compatMode.severity" value="IGNORE"/>
+  <set-property name="gwt.uncaughtexceptionhandler.windowonerror" value="IGNORE"/>
 
   <set-property name="jsinterop.checks" value="ENABLED"/>
+
+  <!-- Limit to one permutation, which we'll assume to be "safari" - this is what chromium registers as as well,
+       and for our purposes firefox will be compatible too.
+   -->
+  <set-property name="user.agent" value="safari" />
 </module>

web/client-api/src/main/java/io/deephaven/web/DeephavenJsApiLinkerTemplate.js

@@ -22,8 +22,6 @@ bindTo($wnd, globalThis);
 var window = $wnd;
 var dh = {};
 $wnd.dh = dh;
-import {dhinternal} from './dh-internal.js';
-$wnd.dhinternal = dhinternal;
 var $gwt_version = "__GWT_VERSION__";
 __JAVASCRIPT_RESULT__
 gwtOnLoad(null, '__MODULE_NAME__', null);