Feature: Upload external datasets (#112)

* add endpoint for uploading external datasets

* chunk segments into payload sizes less than Jetty size limit

* cleanup after dataset creation failure

* fix codeql errors (squashable)

* return external dataset upload errors to client

Author: duranb

package-lock.json

@@ -30,6 +30,7 @@
     "altair-express-middleware": "^5.2.11",
     "cookie-parser": "^1.4.6",
     "cors": "^2.8.5",
+    "csv-parse": "^5.5.6",
     "express": "^4.18.2",
     "express-rate-limit": "^6.7.0",
     "helmet": "^7.0.0",

package.json

@@ -1,5 +1,5 @@
 import type { Algorithm } from 'jsonwebtoken';
-import { GroupRoleMapping } from './packages/auth/types';
+import { GroupRoleMapping } from './types/auth';
 
 export type Env = {
   ALLOWED_ROLES: string[];

src/env.ts

@@ -12,7 +12,7 @@ import initHealthRoutes from './packages/health/health.js';
 import initPlanRoutes from './packages/plan/plan.js';
 import initSwaggerRoutes from './packages/swagger/swagger.js';
 import cookieParser from 'cookie-parser';
-import { AuthAdapter } from './packages/auth/types.js';
+import { AuthAdapter } from './types/auth.js';
 import { NoAuthAdapter } from './packages/auth/adapters/NoAuthAdapter.js';
 import { CAMAuthAdapter } from './packages/auth/adapters/CAMAuthAdapter.js';
 import { validateGroupRoleMappings } from './packages/auth/functions.js';

src/main.ts

@@ -1,7 +1,7 @@
 import { getEnv } from '../../../env.js';
 import { authGroupMappingsExist, generateJwt, getUserRoles, mapGroupsToRoles, syncRolesToDB } from '../functions.js';
 import fetch from 'node-fetch';
-import type { AuthAdapter, AuthResponse, ValidateResponse } from '../types.js';
+import type { AuthAdapter, AuthResponse, ValidateResponse } from '../../../types/auth.js';
 
 import { Request } from 'express';
 

src/packages/auth/adapters/CAMAuthAdapter.ts

@@ -1,4 +1,4 @@
-import type { AuthAdapter, ValidateResponse } from '../types.js';
+import type { AuthAdapter, ValidateResponse } from '../../../types/auth.js';
 
 export const NoAuthAdapter: AuthAdapter = {
   logout: async (): Promise<boolean> => true,

src/packages/auth/adapters/NoAuthAdapter.ts

@@ -12,7 +12,7 @@ import type {
   JwtSecret,
   SessionResponse,
   UserRoles,
-} from './types.js';
+} from '../../types/auth.js';
 import { loginSSO } from './adapters/CAMAuthAdapter.js';
 
 const logger = getLogger('packages/auth/functions');

src/packages/auth/functions.ts

@@ -2,7 +2,7 @@ import type { Express } from 'express';
 import rateLimit from 'express-rate-limit';
 import { getEnv } from '../../env.js';
 import { login, session } from './functions.js';
-import { AuthAdapter } from './types.js';
+import { AuthAdapter } from '../../types/auth.js';
 
 export default (app: Express, auth: AuthAdapter) => {
   const { RATE_LIMITER_LOGIN_MAX } = getEnv();

src/packages/auth/routes.ts

@@ -1,4 +1,19 @@
 export default {
+  ADD_EXTERNAL_DATASET: `#graphql
+    mutation AddExternalDataset(
+      $planId: Int!,
+      $simulationDatasetId: Int,
+      $datasetStart: String!,
+      $profileSet: ProfileSet!) {
+        addExternalDataset(
+          planId: $planId,
+          simulationDatasetId: $simulationDatasetId,
+          datasetStart: $datasetStart,
+          profileSet: $profileSet) {
+          datasetId
+        }
+    }
+  `,
   CREATE_ACTIVITY_DIRECTIVES: `#graphql
     mutation CreateActivityDirectives($activityDirectivesInsertInput: [activity_directive_insert_input!]!) {
       insert_activity_directive(objects: $activityDirectivesInsertInput) {
@@ -50,6 +65,13 @@ export default {
       }
     }
   `,
+  DELETE_EXTERNAL_DATASET: `#graphql
+    mutation DeleteExternalDataset($id: Int!) {
+      delete_dataset_by_pk(id: $id) {
+        id
+      }
+    }
+  `,
   DELETE_PLAN: `#graphql
     mutation DeletePlan($id: Int!) {
       deletePlan: delete_plan_by_pk(id: $id) {
@@ -68,6 +90,13 @@ export default {
       }
     }
   `,
+  EXTEND_EXTERNAL_DATASET: `#graphql
+    mutation ExtendExternalDataset($datasetId: Int!, $profileSet: ProfileSet!) {
+      extendExternalDataset(datasetId: $datasetId, profileSet: $profileSet) {
+        datasetId
+      }
+    }
+  `,
   GET_TAGS: `#graphql
     query GetTags {
       tags(order_by: { name: desc })  {