Simplify benchmark test suite (#959)

Author: ardatan

.github/workflows/bench.yml

@@ -17,13 +17,7 @@ jobs:
     strategy:
       matrix:
         e2e_runner: [node, bun]
-        products_size:
-          # The results don't change that much based on the list size,
-          # But later we might want them back
-          # - 10
-          # - 100
-          - 1000
-    name: ${{matrix.e2e_runner}} / ${{matrix.products_size}} items
+    name: ${{matrix.e2e_runner}}
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -35,5 +29,4 @@ jobs:
       - name: Bench
         run: ${{matrix.e2e_runner == 'bun' && './node_modules/.bin/bun' || 'yarn'}} bench
         env:
-          PRODUCTS_SIZE: ${{matrix.products_size}}
           E2E_GATEWAY_RUNNER: ${{matrix.e2e_runner}}

bench/federation/federation.bench.ts

@@ -32,8 +32,10 @@ function memoize1<T extends (...args: any) => any>(fn: T): T {
 
 describe('Federation', async () => {
   const { fs } = createTenv(__dirname);
-  const { query, operationName, result, supergraph } =
-    createExampleSetup(__dirname);
+  const { query, operationName, result, supergraph } = createExampleSetup(
+    __dirname,
+    1000,
+  );
   const services: Record<string, { schema: GraphQLSchema }> = {
     accounts,
     inventory,
@@ -75,9 +77,21 @@ describe('Federation', async () => {
     throw new Error(`Operation ${operationName} not found`);
   }
 
-  let apolloGW: ApolloGateway;
-  let apolloGWSchema: GraphQLSchema;
-  let apolloGWExecutor: ApolloGateway['executor'];
+  const apolloGW = new ApolloGateway({
+    logger: dummyLogger,
+    supergraphSdl,
+    buildService({ name }) {
+      const lowercasedName = name.toLowerCase();
+      const service = services[lowercasedName];
+      if (!service) {
+        throw new Error(`Service ${name} not found`);
+      }
+      return new LocalGraphQLDataSource(service.schema);
+    },
+  });
+
+  let { schema: apolloGWSchema, executor: apolloGWExecutor } =
+    await apolloGW.load();
 
   const schemaHash: string & { __fauxpaque: 'SchemaHash' } = Object.assign(
     new String(supergraphSdl) as string,
@@ -111,32 +125,20 @@ describe('Federation', async () => {
         },
       );
     },
-    {
-      async setup() {
-        apolloGW = new ApolloGateway({
-          logger: dummyLogger,
-          supergraphSdl,
-          buildService({ name }) {
-            const lowercasedName = name.toLowerCase();
-            const service = services[lowercasedName];
-            if (!service) {
-              throw new Error(`Service ${name} not found`);
-            }
-            return new LocalGraphQLDataSource(service.schema);
-          },
-        });
-        const { schema, executor } = await apolloGW.load();
-        apolloGWSchema = schema;
-        apolloGWExecutor = executor;
-      },
-      teardown() {
-        return apolloGW.stop();
-      },
-      ...benchConfig,
-    },
+    benchConfig,
   );
 
-  let stitchedSchema: GraphQLSchema;
+  const stitchedSchema = getStitchedSchemaFromSupergraphSdl({
+    supergraphSdl,
+    onSubschemaConfig(subschemaConfig) {
+      const lowercasedName = subschemaConfig.name.toLowerCase();
+      const service = services[lowercasedName];
+      if (!service) {
+        throw new Error(`Service ${subschemaConfig.name} not found`);
+      }
+      subschemaConfig.executor = createDefaultExecutor(service.schema);
+    },
+  });
 
   bench(
     'Stitching',
@@ -153,21 +155,6 @@ describe('Federation', async () => {
           expect(response).toEqual(result);
         },
       ),
-    {
-      setup() {
-        stitchedSchema = getStitchedSchemaFromSupergraphSdl({
-          supergraphSdl,
-          onSubschemaConfig(subschemaConfig) {
-            const lowercasedName = subschemaConfig.name.toLowerCase();
-            const service = services[lowercasedName];
-            if (!service) {
-              throw new Error(`Service ${subschemaConfig.name} not found`);
-            }
-            subschemaConfig.executor = createDefaultExecutor(service.schema);
-          },
-        });
-      },
-      ...benchConfig,
-    },
+    benchConfig,
   );
 });

e2e/federation-example/federation-example.bench.ts

@@ -1,84 +1,60 @@
-import { ApolloGateway } from '@apollo/gateway';
-import { ApolloServer } from '@apollo/server';
-import { startStandaloneServer } from '@apollo/server/standalone';
-import { createExampleSetup, createTenv, Gateway } from '@internal/e2e';
-import { benchConfig } from '@internal/testing';
+import {
+  createExampleSetup,
+  createTenv,
+  Gateway,
+  Service,
+} from '@internal/e2e';
+import { benchConfig, getLocalhost } from '@internal/testing';
 import { fetch } from '@whatwg-node/fetch';
 import { bench, describe, expect } from 'vitest';
 
 describe('Gateway', async () => {
-  const { gateway, fs } = createTenv(__dirname);
-  const example = createExampleSetup(__dirname);
+  const { gateway, service } = createTenv(__dirname);
+  const example = createExampleSetup(__dirname, 1000);
 
   const supergraph = await example.supergraph();
-  const supergraphSdl = await fs.read(supergraph);
 
-  let apolloGw: ApolloServer;
-  let apolloGwUrl: string;
-  let ctrl: AbortController;
-  bench(
-    'Apollo Gateway',
-    async () => {
-      const res = await fetch(`${apolloGwUrl}/graphql`, {
-        method: 'POST',
-        headers: {
-          'Content-Type': 'application/json',
-        },
-        body: JSON.stringify({
-          query: example.query,
-        }),
-        signal: ctrl.signal,
-      });
-      const data = await res.json();
-      expect(data).toEqual({
-        data: expect.any(Object),
-      });
-    },
-    {
-      async setup() {
-        ctrl = new AbortController();
-        apolloGw = new ApolloServer({
-          gateway: new ApolloGateway({
-            supergraphSdl,
-          }),
-        });
-        const { url } = await startStandaloneServer(apolloGw, {
-          listen: { port: 0 },
-        });
-        apolloGwUrl = url;
+  const gateways: Record<string, Gateway | Service> = {
+    'Apollo Gateway': await service('apollo-gateway', {
+      env: {
+        SUPERGRAPH: supergraph,
       },
-      teardown() {
-        ctrl.abort();
-        return apolloGw.stop();
+    }),
+    'Hive Gateway': await gateway({
+      supergraph,
+      args: ['--jit'],
+      env: {
+        JIT: 'true',
+        NODE_ENV: 'production',
       },
-      ...benchConfig,
-    },
-  );
+    }),
+  };
 
-  let hiveGw: Gateway;
-  bench(
-    'Hive Gateway',
-    async () => {
-      const res = await hiveGw.execute({
-        query: example.query,
-      });
-      expect(res).toEqual(example.result);
-    },
-    {
-      async setup() {
-        hiveGw = await gateway({
-          supergraph,
-          args: ['--jit'],
-          env: {
-            NODE_ENV: 'production',
-            JIT: 'true',
+  for (const gwName in gateways) {
+    const gw = gateways[gwName];
+    if (!gw) {
+      throw new Error(`Gateway ${gwName} not found`);
+    }
+    const gwPort = gw.port;
+    const gwHost = await getLocalhost(gwPort, gw.protocol);
+    const gwUrl = `${gwHost}:${gwPort}/graphql`;
+    const body = JSON.stringify({
+      query: example.query,
+    });
+    bench(
+      gwName,
+      async () => {
+        const response = await fetch(gwUrl, {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
           },
+          body,
         });
+        const data = await response.json();
+        expect(data).toEqual(example.result);
       },
-      async teardown() {
-        return hiveGw[Symbol.asyncDispose]();
-      },
-      ...benchConfig,
-    },
-  );
+      benchConfig,
+    );
+  }
 });

e2e/federation-example/services/apollo-gateway.ts

@@ -0,0 +1,30 @@
+import { readFileSync } from 'fs';
+import { ApolloGateway } from '@apollo/gateway';
+import { ApolloServer } from '@apollo/server';
+import { startStandaloneServer } from '@apollo/server/standalone';
+import { Opts } from '@internal/testing';
+
+const opts = Opts(process.argv);
+const port = opts.getServicePort('apollo-gateway');
+
+async function main() {
+  const supergraph = process.env['SUPERGRAPH'];
+  if (!supergraph) {
+    throw new Error('SUPERGRAPH env var is required');
+  }
+  const supergraphSdl = readFileSync(supergraph, 'utf-8');
+  const server = new ApolloServer({
+    gateway: new ApolloGateway({
+      supergraphSdl,
+    }),
+  });
+  const { url } = await startStandaloneServer(server, {
+    listen: { port },
+  });
+  console.log(`🚀 Gateway ready at ${url}`);
+}
+
+main().catch((error) => {
+  console.error(error);
+  process.exit(1);
+});