Merge pull request #1 from ddelgrosso1/tm-performance-framework

feat: add directory generator to performance test framework

Author: ddelgrosso1

internal-tooling/performPerformanceTest.ts

@@ -15,22 +15,25 @@
  */
 
 import yargs from 'yargs';
-import * as uuid from 'uuid';
-import {execSync} from 'child_process';
 import {unlinkSync} from 'fs';
 import {Storage} from '../src';
 import {performance} from 'perf_hooks';
 // eslint-disable-next-line node/no-unsupported-features/node-builtins
 import {parentPort} from 'worker_threads';
 import path = require('path');
+import {
+  BLOCK_SIZE_IN_BYTES,
+  DEFAULT_LARGE_FILE_SIZE_BYTES,
+  DEFAULT_SMALL_FILE_SIZE_BYTES,
+  generateRandomFile,
+  generateRandomFileName,
+  randomInteger,
+} from './performanceUtils';
 
 const TEST_NAME_STRING = 'nodejs-perf-metrics';
 const DEFAULT_NUMBER_OF_WRITES = 1;
 const DEFAULT_NUMBER_OF_READS = 3;
 const DEFAULT_BUCKET_NAME = 'nodejs-perf-metrics';
-const DEFAULT_SMALL_FILE_SIZE_BYTES = 5120;
-const DEFAULT_LARGE_FILE_SIZE_BYTES = 2.147e9;
-const BLOCK_SIZE_IN_BYTES = 1024;
 const NODE_DEFAULT_HIGHWATER_MARK_BYTES = 16384;
 
 export interface TestResult {
@@ -46,20 +49,6 @@ export interface TestResult {
   status: '[OK]';
 }
 
-/**
- * Create a uniformly distributed random integer beween the inclusive min and max provided.
- *
- * @param {number} minInclusive lower bound (inclusive) of the range of random integer to return.
- * @param {number} maxInclusive upper bound (inclusive) of the range of random integer to return.
- * @returns {number} returns a random integer between minInclusive and maxInclusive
- */
-const randomInteger = (minInclusive: number, maxInclusive: number) => {
-  // Utilizing Math.random will generate uniformly distributed random numbers.
-  return (
-    Math.floor(Math.random() * (maxInclusive - minInclusive + 1)) + minInclusive
-  );
-};
-
 const argv = yargs(process.argv.slice(2))
   .options({
     bucket: {type: 'string', default: DEFAULT_BUCKET_NAME},
@@ -85,8 +74,8 @@ async function main() {
  */
 async function performWriteReadTest(): Promise<TestResult[]> {
   const results: TestResult[] = [];
-  const fileName = generateRandomFileName();
-  const sizeInBytes = generateRandomFile(fileName);
+  const fileName = generateRandomFileName(TEST_NAME_STRING);
+  const sizeInBytes = generateRandomFile(fileName, argv.small, argv.large);
   const checkType = randomInteger(0, 2);
 
   const stg = new Storage({
@@ -159,7 +148,7 @@ async function performWriteReadTest(): Promise<TestResult[]> {
       status: '[OK]',
     };
 
-    const destinationFileName = generateRandomFileName();
+    const destinationFileName = generateRandomFileName(TEST_NAME_STRING);
     const destination = path.join(__dirname, destinationFileName);
     if (checkType === 0) {
       start = performance.now();
@@ -186,31 +175,6 @@ async function performWriteReadTest(): Promise<TestResult[]> {
   return results;
 }
 
-/**
- * Creates a file with a size between the small (default 5120 bytes) and large (2.147e9 bytes) parameters.
- * The file is filled with random data.
- *
- * @param {string} fileName name of the file to generate.
- * @returns {number} the size of the file generated.
- */
-function generateRandomFile(fileName: string) {
-  const fileSizeBytes = randomInteger(argv.small, argv.large);
-  const numberNeeded = Math.ceil(fileSizeBytes / BLOCK_SIZE_IN_BYTES);
-  const cmd = `dd if=/dev/urandom of=${__dirname}/${fileName} bs=${BLOCK_SIZE_IN_BYTES} count=${numberNeeded} status=none iflag=fullblock`;
-  execSync(cmd);
-
-  return fileSizeBytes;
-}
-
-/**
- * Creates a random file name by appending a UUID to the TEST_NAME_STRING.
- *
- * @returns {string} random file name that was generated.
- */
-function generateRandomFileName(): string {
-  return `${TEST_NAME_STRING}.${uuid.v4()}`;
-}
-
 /**
  * Deletes the file specified by the fileName parameter.
  *

internal-tooling/performanceUtils.ts

@@ -0,0 +1,110 @@
+/*!
+ * Copyright 2022 Google LLC. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {execSync} from 'child_process';
+import {mkdirSync} from 'fs';
+import path = require('path');
+import * as uuid from 'uuid';
+
+export const BLOCK_SIZE_IN_BYTES = 1024;
+export const DEFAULT_SMALL_FILE_SIZE_BYTES = 5120;
+export const DEFAULT_LARGE_FILE_SIZE_BYTES = 2.147e9;
+
+const CREATE_DIRECTORY = 1;
+
+/**
+ * Create a uniformly distributed random integer beween the inclusive min and max provided.
+ *
+ * @param {number} minInclusive lower bound (inclusive) of the range of random integer to return.
+ * @param {number} maxInclusive upper bound (inclusive) of the range of random integer to return.
+ * @returns {number} returns a random integer between minInclusive and maxInclusive
+ */
+export function randomInteger(minInclusive: number, maxInclusive: number) {
+  // Utilizing Math.random will generate uniformly distributed random numbers.
+  return (
+    Math.floor(Math.random() * (maxInclusive - minInclusive + 1)) + minInclusive
+  );
+}
+
+/**
+ * Creates a random file name by appending a UUID to the baseName.
+ *
+ * @param {string} baseName the base file name. A random uuid will be appended to this value.
+ *
+ * @returns {string} random file name that was generated.
+ */
+export function generateRandomFileName(baseName: string): string {
+  return `${baseName}.${uuid.v4()}`;
+}
+
+/**
+ * Creates a file with a size between the small (default 5120 bytes) and large (2.147e9 bytes) parameters.
+ * The file is filled with random data.
+ *
+ * @param {string} fileName name of the file to generate.
+ * @param {number} fileSizeLowerBoundBytes minimum size of file to generate.
+ * @param {number} fileSizeUpperBoundBytes maximum size of file to generate.
+ * @param {string} currentDirectory the directory in which to generate the file.
+ *
+ * @returns {number} the size of the file generated.
+ */
+export function generateRandomFile(
+  fileName: string,
+  fileSizeLowerBoundBytes: number = DEFAULT_SMALL_FILE_SIZE_BYTES,
+  fileSizeUpperBoundBytes: number = DEFAULT_LARGE_FILE_SIZE_BYTES,
+  currentDirectory: string = __dirname
+) {
+  const fileSizeBytes = randomInteger(
+    fileSizeLowerBoundBytes,
+    fileSizeUpperBoundBytes
+  );
+  const numberNeeded = Math.ceil(fileSizeBytes / BLOCK_SIZE_IN_BYTES);
+  const cmd = `dd if=/dev/urandom of=${currentDirectory}/${fileName} bs=${BLOCK_SIZE_IN_BYTES} count=${numberNeeded} status=none iflag=fullblock`;
+  execSync(cmd);
+
+  return fileSizeBytes;
+}
+
+/**
+ * Creates a random directory structure consisting of subdirectories and random files.
+ *
+ * @param {number} maxObjects the total number of subdirectories and files to generate.
+ * @param {string} baseName the starting directory under which everything else is added. File names will have this value prepended.
+ * @param {number} fileSizeLowerBoundBytes minimum size of file to generate.
+ * @param {number} fileSizeUpperBoundBytes maximum size of file to generate.
+ */
+export function generateRandomDirectoryStructure(
+  maxObjects: number,
+  baseName: string,
+  fileSizeLowerBoundBytes: number = DEFAULT_SMALL_FILE_SIZE_BYTES,
+  fileSizeUpperBoundBytes: number = DEFAULT_LARGE_FILE_SIZE_BYTES
+) {
+  let curPath = baseName;
+  for (let i = 0; i < maxObjects; i++) {
+    const dirOrFile = randomInteger(0, 1);
+    if (dirOrFile === CREATE_DIRECTORY) {
+      curPath = path.join(curPath, uuid.v4());
+      mkdirSync(curPath, {recursive: true});
+    } else {
+      generateRandomFile(
+        generateRandomFileName(baseName),
+        fileSizeLowerBoundBytes,
+        fileSizeUpperBoundBytes,
+        curPath
+      );
+    }
+  }
+}