|
| 1 | +#!/bin/bash |
| 2 | +# |
| 3 | +# Copyright 2018-2023 contributors to the Marquez project |
| 4 | +# SPDX-License-Identifier: Apache-2.0 |
| 5 | +# |
| 6 | +# A script used in CI to load test HTTP API server by: |
| 7 | +# (1) Starting HTTP API server |
| 8 | +# (2) Generating random dataset, job, and run metadata |
| 9 | +# (3) Running load test using k6 |
| 10 | +# (4) Writing load test results to 'k6/results' for analysis |
| 11 | +# |
| 12 | +# Usage: $ ./api-load-test.sh |
| 13 | + |
| 14 | +set -e |
| 15 | + |
| 16 | +# Build version of Marquez |
| 17 | +readonly MARQUEZ_VERSION="0.30.0-SNAPSHOT" |
| 18 | +# Fully qualified path to marquez.jar |
| 19 | +readonly MARQUEZ_JAR="api/build/libs/marquez-api-${MARQUEZ_VERSION}.jar" |
| 20 | + |
| 21 | +readonly MARQUEZ_HOST="localhost" |
| 22 | +readonly MARQUEZ_ADMIN_PORT=5001 # Use default 'dev' admin port |
| 23 | +readonly MARQUEZ_URL="http://${MARQUEZ_HOST}:${MARQUEZ_ADMIN_PORT}" |
| 24 | +readonly MARQUEZ_DB="marquez-db" |
| 25 | + |
| 26 | +readonly METADATA_FILE="api/load-testing/metadata.json" |
| 27 | +readonly METADATA_STATS_QUERY=$(cat <<-END |
| 28 | + SELECT run_uuid,COUNT(*) |
| 29 | + FROM lineage_events |
| 30 | + GROUP BY run_uuid; |
| 31 | +END |
| 32 | +) |
| 33 | + |
| 34 | +log() { |
| 35 | + echo -e "\033[1m>>\033[0m ${1}" |
| 36 | +} |
| 37 | + |
| 38 | +cpu_and_mem_info() { |
| 39 | + log "CPU info:" |
| 40 | + cat /proc/cpuinfo |
| 41 | + log "MEM info:" |
| 42 | + cat /proc/meminfo |
| 43 | +} |
| 44 | + |
| 45 | +ol_events_stats() { |
| 46 | + # Query db for OL events stats |
| 47 | + log "load test metadata stats:" |
| 48 | + docker exec "${MARQUEZ_DB}" \ |
| 49 | + psql -U marquez -c "${METADATA_STATS_QUERY}" |
| 50 | +} |
| 51 | + |
| 52 | +# Change working directory to project root |
| 53 | +project_root=$(git rev-parse --show-toplevel) |
| 54 | +cd "${project_root}" |
| 55 | + |
| 56 | +# (1) Start db |
| 57 | +log "start db:" |
| 58 | +docker-compose -f docker-compose.db.yml up --detach |
| 59 | + |
| 60 | +# (2) Build HTTP API server |
| 61 | +log "build http API server..." |
| 62 | +./gradlew --no-daemon :api:build -x test > /dev/null 2>&1 |
| 63 | + |
| 64 | +# (3) Start HTTP API server |
| 65 | +log "start http API server..." |
| 66 | +mkdir marquez && \ |
| 67 | + java -jar "${MARQUEZ_JAR}" server marquez.dev.yml > marquez/http.log 2>&1 & |
| 68 | + |
| 69 | +# (4) Wait for HTTP API server |
| 70 | +log "waiting for http API server (${MARQUEZ_URL})..." |
| 71 | +until curl --output /dev/null --silent --head --fail "${MARQUEZ_URL}/ping"; do |
| 72 | + sleep 5 |
| 73 | +done |
| 74 | +# When available, print status |
| 75 | +log "http API server is ready!" |
| 76 | + |
| 77 | +# (5) Use metadata command to generate random dataset, job, and run metadata |
| 78 | +log "generate load test metadata (${METADATA_FILE}):" |
| 79 | +java -jar "${MARQUEZ_JAR}" metadata --runs 10 --bytes-per-event 16384 --output "${METADATA_FILE}" |
| 80 | + |
| 81 | +# Display CPU/MEM |
| 82 | +cpu_and_mem_info |
| 83 | + |
| 84 | +# (6) Run load test |
| 85 | +log "start load test:" |
| 86 | +mkdir -p k6/results && \ |
| 87 | + k6 run --vus 25 --duration 30s api/load-testing/http.js \ |
| 88 | + --out json=k6/results/full.json --summary-export=k6/results/summary.json |
| 89 | + |
| 90 | +# Display OL event stats |
| 91 | +ol_events_stats |
| 92 | + |
| 93 | +echo "DONE!" |
0 commit comments