APM Cypress Testing + Prometheus #5321
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Dashboards observability plugin E2E test | |
| on: [pull_request, push] | |
| env: | |
| PLUGIN_NAME: dashboards-observability | |
| OPENSEARCH_DASHBOARDS_VERSION: 'main' | |
| OPENSEARCH_VERSION: '3.6.0' | |
| OPENSEARCH_PLUGIN_VERSION: '3.6.0.0-SNAPSHOT' | |
| jobs: | |
| tests: | |
| name: Run test group of ${{ matrix.testgroups }} | |
| env: | |
| # Prevents extra Cypress installation progress messages | |
| CI: 1 | |
| # Avoid warnings like "tput: No value for $TERM and no -T specified" | |
| TERM: xterm | |
| WORKING_DIR: ${{ matrix.working_directory }}. | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] | |
| java: [21] | |
| testgroups: | |
| [ | |
| app_analytics_test, | |
| event_analytics_test, | |
| integrations_test, | |
| metrics_analytics_test, | |
| notebooks_test, | |
| panels_test, | |
| trace_analytics_test, | |
| apm_test, | |
| ] | |
| include: | |
| - os: ubuntu-latest | |
| cypress_cache_folder: ~/.cache/Cypress | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Set up Java 21 | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: 'corretto' | |
| java-version: '21' | |
| - name: Get latest Job Scheduler snapshot version | |
| id: job-scheduler-snapshot | |
| run: | | |
| METADATA_URL="https://ci.opensearch.org/ci/dbc/snapshots/maven/org/opensearch/plugin/opensearch-job-scheduler/${{ env.OPENSEARCH_PLUGIN_VERSION }}/maven-metadata.xml" | |
| SNAPSHOT_VERSION=$(curl -s $METADATA_URL | grep '<value>' | head -1 | sed 's/.*<value>\(.*\)<\/value>.*/\1/') | |
| echo "version=$SNAPSHOT_VERSION" >> $GITHUB_OUTPUT | |
| echo "Job Scheduler snapshot version: $SNAPSHOT_VERSION" | |
| - name: Download Job Scheduler artifact | |
| uses: suisei-cn/actions-download-file@v1.4.0 | |
| with: | |
| url: https://ci.opensearch.org/ci/dbc/snapshots/maven/org/opensearch/plugin/opensearch-job-scheduler/${{ env.OPENSEARCH_PLUGIN_VERSION }}/opensearch-job-scheduler-${{ steps.job-scheduler-snapshot.outputs.version }}.zip | |
| target: plugin-artifacts/ | |
| filename: opensearch-job-scheduler.zip | |
| - name: Get latest Observability snapshot version | |
| id: observability-snapshot | |
| run: | | |
| METADATA_URL="https://ci.opensearch.org/ci/dbc/snapshots/maven/org/opensearch/plugin/opensearch-observability/${{ env.OPENSEARCH_PLUGIN_VERSION }}/maven-metadata.xml" | |
| SNAPSHOT_VERSION=$(curl -s $METADATA_URL | grep '<value>' | head -1 | sed 's/.*<value>\(.*\)<\/value>.*/\1/') | |
| echo "version=$SNAPSHOT_VERSION" >> $GITHUB_OUTPUT | |
| echo "Observability snapshot version: $SNAPSHOT_VERSION" | |
| - name: Download observability artifact | |
| uses: suisei-cn/actions-download-file@v1.4.0 | |
| with: | |
| url: https://ci.opensearch.org/ci/dbc/snapshots/maven/org/opensearch/plugin/opensearch-observability/${{ env.OPENSEARCH_PLUGIN_VERSION }}/opensearch-observability-${{ steps.observability-snapshot.outputs.version }}.zip | |
| target: plugin-artifacts/ | |
| filename: observability.zip | |
| - name: Get latest SQL Plugin snapshot version | |
| id: sql-snapshot | |
| run: | | |
| METADATA_URL="https://ci.opensearch.org/ci/dbc/snapshots/maven/org/opensearch/plugin/opensearch-sql-plugin/${{ env.OPENSEARCH_PLUGIN_VERSION }}/maven-metadata.xml" | |
| SNAPSHOT_VERSION=$(curl -s $METADATA_URL | grep '<value>' | head -1 | sed 's/.*<value>\(.*\)<\/value>.*/\1/') | |
| echo "version=$SNAPSHOT_VERSION" >> $GITHUB_OUTPUT | |
| echo "SQL Plugin snapshot version: $SNAPSHOT_VERSION" | |
| - name: Download SQL artifact | |
| uses: suisei-cn/actions-download-file@v1.4.0 | |
| with: | |
| url: https://ci.opensearch.org/ci/dbc/snapshots/maven/org/opensearch/plugin/opensearch-sql-plugin/${{ env.OPENSEARCH_PLUGIN_VERSION }}/opensearch-sql-plugin-${{ steps.sql-snapshot.outputs.version }}.zip | |
| target: plugin-artifacts/ | |
| filename: sql.zip | |
| - name: Download OpenSearch | |
| uses: peternied/download-file@v2 | |
| with: | |
| url: https://artifacts.opensearch.org/snapshots/core/opensearch/${{ env.OPENSEARCH_VERSION }}-SNAPSHOT/opensearch-min-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT-linux-x64-latest.tar.gz | |
| - name: Extract OpenSearch | |
| run: | | |
| tar -xzf opensearch-*.tar.gz | |
| rm -f opensearch-*.tar.gz | |
| shell: bash | |
| - name: Install job scheduler plugin | |
| run: | | |
| /bin/bash -c "yes | ./opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT/bin/opensearch-plugin install file:$(pwd)/plugin-artifacts/opensearch-job-scheduler.zip" | |
| shell: bash | |
| - name: Install observability plugin | |
| run: | | |
| /bin/bash -c "yes | ./opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT/bin/opensearch-plugin install file:$(pwd)/plugin-artifacts/observability.zip" | |
| shell: bash | |
| - name: Install SQL plugin | |
| run: | | |
| /bin/bash -c "yes | ./opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT/bin/opensearch-plugin install file:$(pwd)/plugin-artifacts/sql.zip" | |
| shell: bash | |
| - name: Configure OpenSearch for auto index creation | |
| run: | | |
| cat << 'EOF' >> ./opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT/config/opensearch.yml | |
| action.auto_create_index: true | |
| EOF | |
| shell: bash | |
| - name: Run OpenSearch | |
| run: /bin/bash -c "./opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT/bin/opensearch -Ecluster.routing.allocation.disk.threshold_enabled=false -Ecluster.routing.allocation.disk.watermark.low=1gb -Ecluster.routing.allocation.disk.watermark.high=500mb -Ecluster.routing.allocation.disk.watermark.flood_stage=100mb -Ecluster.blocks.read_only_allow_delete=false &" | |
| shell: bash | |
| - name: Checkout OpenSearch Dashboards | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: opensearch-project/Opensearch-Dashboards | |
| ref: ${{ env.OPENSEARCH_DASHBOARDS_VERSION }} | |
| path: OpenSearch-Dashboards | |
| - name: Checkout dashboards observability | |
| uses: actions/checkout@v4 | |
| with: | |
| path: OpenSearch-Dashboards/plugins/dashboards-observability | |
| - name: Get node and yarn versions | |
| working-directory: ${{ env.WORKING_DIR }} | |
| id: versions_step | |
| run: | | |
| echo "::set-output name=node_version::$(cat ./OpenSearch-Dashboards/.nvmrc | cut -d"." -f1)" | |
| echo "::set-output name=yarn_version::$(node -p "(require('./OpenSearch-Dashboards/package.json').engines.yarn).match(/[.0-9]+/)[0]")" | |
| - name: Setup node | |
| uses: actions/setup-node@v1 | |
| with: | |
| node-version: ${{ steps.versions_step.outputs.node_version }} | |
| registry-url: 'https://registry.npmjs.org' | |
| - name: Configure OpenSearch Dashboards | |
| run: | | |
| rm -rf ./config/opensearch_dashboards.yml | |
| cat << 'EOT' > ./config/opensearch_dashboards.yml | |
| server.host: "0.0.0.0" | |
| home.disableWelcomeScreen: true | |
| EOT | |
| working-directory: OpenSearch-Dashboards | |
| - name: Install correct yarn version for OpenSearch Dashboards | |
| id: setup-yarn | |
| run: | | |
| npm uninstall -g yarn | |
| echo "Installing yarn ${{ steps.versions_step.outputs.yarn_version }}" | |
| npm i -g yarn@${{ steps.versions_step.outputs.yarn_version }} | |
| - name: Get Cypress version | |
| id: cypress_version | |
| run: | | |
| cd ./OpenSearch-Dashboards/plugins/dashboards-observability | |
| echo "::set-output name=cypress_version::$(cat ./package.json | jq '.dependencies.cypress' | tr -d '"')" | |
| - name: Yarn Cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| OpenSearch-Dashboards/**/target | |
| OpenSearch-Dashboards/**/node_modules | |
| key: ${{ runner.OS }}-build-${{ hashFiles('OpenSearch-Dashboards/**/yarn.lock') }} | |
| - name: Cache Cypress binary | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ matrix.cypress_cache_folder }} | |
| key: ${{ runner.OS }}-cypress-${{ steps.cypress_version.outputs.cypress_version }} | |
| restore-keys: | | |
| ${{ runner.OS }}-cypress- | |
| - name: Bootstrap OpenSearch Dashboards | |
| run: | | |
| cd OpenSearch-Dashboards | |
| yarn osd bootstrap --single-version=loose | |
| - name: Install/Verify Cypress binary | |
| run: | | |
| cd ./OpenSearch-Dashboards | |
| npx cypress install | |
| npx cypress verify | |
| - name: Compile OpenSearch Dashboards | |
| run: | | |
| cd OpenSearch-Dashboards | |
| node scripts/build_opensearch_dashboards_platform_plugins --no-examples --workers=10 --verbose | |
| - name: Start APM Metrics Server and Prometheus | |
| if: matrix.testgroups == 'apm_test' | |
| run: | | |
| cd OpenSearch-Dashboards/plugins/dashboards-observability | |
| # Download and extract Prometheus | |
| echo "Downloading Prometheus..." | |
| cd ../../.. | |
| PROM_VERSION="3.8.1" | |
| curl -sL https://github.com/prometheus/prometheus/releases/download/v${PROM_VERSION}/prometheus-${PROM_VERSION}.linux-amd64.tar.gz | tar xzf - | |
| # Add Prometheus to PATH | |
| export PATH="$(pwd)/prometheus-${PROM_VERSION}.linux-amd64:$PATH" | |
| # Verify Prometheus is available | |
| prometheus --version | |
| cd OpenSearch-Dashboards/plugins/dashboards-observability | |
| # Start metrics server in background | |
| echo "Starting APM metrics server on port 8080..." | |
| node .cypress/utils/metrics_server.js > /tmp/metrics-server.log 2>&1 & | |
| METRICS_SERVER_PID=$! | |
| echo "Metrics server PID: $METRICS_SERVER_PID" | |
| # Wait for metrics server to be ready | |
| echo "Waiting for metrics server to start..." | |
| for i in {1..15}; do | |
| if curl -s http://localhost:8080/metrics > /dev/null 2>&1; then | |
| echo "Metrics server is ready!" | |
| break | |
| fi | |
| echo "Attempt $i: Metrics server not ready yet..." | |
| sleep 1 | |
| done | |
| # Start Prometheus (will scrape metrics server every 10 seconds) | |
| echo "Starting Prometheus..." | |
| prometheus \ | |
| --config.file=./.cypress/fixtures/prometheus/prometheus.yml \ | |
| --storage.tsdb.path=/tmp/prometheus-apm-data \ | |
| --web.listen-address=:9090 \ | |
| --web.enable-lifecycle \ | |
| --web.enable-admin-api > /tmp/prometheus.log 2>&1 & | |
| PROMETHEUS_PID=$! | |
| echo "Prometheus PID: $PROMETHEUS_PID" | |
| # Wait for Prometheus to be ready | |
| echo "Waiting for Prometheus to start..." | |
| for i in {1..30}; do | |
| if curl -s http://localhost:9090/-/ready > /dev/null 2>&1; then | |
| echo "Prometheus is ready!" | |
| break | |
| fi | |
| echo "Attempt $i: Prometheus not ready yet..." | |
| sleep 2 | |
| done | |
| # Wait 3 minutes for Prometheus to accumulate scrapes | |
| # This ensures sum_over_time queries have enough data in the lookback window | |
| echo "Waiting 3 minutes for Prometheus to accumulate metric scrapes..." | |
| sleep 180 | |
| # Verify metrics are available | |
| echo "Verifying metrics are available..." | |
| QUERY='fault{remoteService=""}' | |
| RESPONSE=$(curl -s "http://localhost:9090/api/v1/query" --data-urlencode "query=$QUERY") | |
| RESULT_COUNT=$(echo "$RESPONSE" | jq '.data.result | length') | |
| echo "Instant query returned $RESULT_COUNT fault metric series" | |
| if [ "$RESULT_COUNT" -eq "0" ]; then | |
| echo "⚠️ Warning: No fault metrics found after waiting" | |
| else | |
| echo "✓ Metrics are available for testing" | |
| fi | |
| shell: bash | |
| - name: Run OpenSearch Dashboards server | |
| run: | | |
| cd OpenSearch-Dashboards | |
| if [ "${{ matrix.testgroups }}" = "apm_test" ]; then | |
| # APM tests require workspace, query enhancements, and dataset management features | |
| nohup yarn start --no-base-path --no-watch \ | |
| --home.disableExperienceModal=true \ | |
| --uiSettings.overrides["query:enhancements:enabled"]=true \ | |
| --data_source.enabled=true \ | |
| --workspace.enabled=true \ | |
| --explore.enabled=true \ | |
| --explore.discoverTraces.enabled=true \ | |
| --datasetManagement.enabled=true | tee dashboard.log & | |
| else | |
| # Other tests use minimal configuration | |
| nohup yarn start --no-base-path --no-watch --home.disableExperienceModal=true | tee dashboard.log & | |
| fi | |
| - name: Check If OpenSearch Dashboards Is Ready | |
| if: ${{ runner.os == 'Linux' }} | |
| run: | | |
| cd ./OpenSearch-Dashboards | |
| if timeout 1200 grep -q "http server running" <(tail -n +1 -f dashboard.log); then | |
| echo "OpenSearch Dashboards started successfully." | |
| else | |
| echo "Timeout of 1200 seconds reached. OpenSearch Dashboards did not start successfully." | |
| echo "Last 100 lines of dashboard.log for debugging:" | |
| tail -100 dashboard.log | |
| exit 1 | |
| fi | |
| - name: Upload Dashboards logs | |
| uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: dashboard.logs | |
| path: OpenSearch-Dashboards/dashboard.log | |
| - name: Reset npm's script shell | |
| if: ${{ matrix.os == 'windows-latest' }} | |
| run: | | |
| npm config delete script-shell | |
| - name: Run Cypress tests for test group of ${{ matrix.testgroups }} | |
| run: | | |
| cd ./OpenSearch-Dashboards/plugins/dashboards-observability | |
| yarn cypress:run --headless --spec '.cypress/integration/${{ matrix.testgroups }}/*' | |
| env: | |
| PROMETHEUS_CONNECTION_URL: ${{ matrix.testgroups == 'apm_test' && 'http://localhost:9090' || '' }} | |
| - name: Capture failure screenshots | |
| uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: cypress-screenshots-${{ matrix.os }}-${{ matrix.testgroups }} | |
| path: OpenSearch-Dashboards/plugins/dashboards-observability/.cypress/screenshots | |
| - name: Capture test video | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: cypress-videos-${{ matrix.os }}-${{ matrix.testgroups }} | |
| path: OpenSearch-Dashboards/plugins/dashboards-observability/.cypress/videos |