Run MCP smoke tests with AI CLI tools #31
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: MCP AI Smoke | |
| on: | |
| pull_request: | |
| types: [opened, synchronize] | |
| workflow_dispatch: | |
| inputs: | |
| gemini_model: | |
| description: Gemini model | |
| required: false | |
| default: gemini-2.5-flash | |
| codex_model: | |
| description: Codex model | |
| required: false | |
| default: gpt-5-mini | |
| claude_model: | |
| description: Claude model | |
| required: false | |
| default: sonnet | |
| permissions: | |
| actions: read | |
| checks: none | |
| contents: read | |
| deployments: none | |
| issues: none | |
| packages: none | |
| pull-requests: read | |
| repository-projects: none | |
| security-events: none | |
| statuses: none | |
| concurrency: | |
| group: mcp-ai-smoke-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| PLUGIN_NAME: McpServer | |
| MATOMO_DIR: ${{ github.workspace }}/matomo | |
| BASE_URL: http://127.0.0.1:8000 | |
| MYSQL_HOST: 127.0.0.1 | |
| MYSQL_PORT: 3306 | |
| MYSQL_USER: root | |
| MYSQL_PASSWORD: root | |
| MYSQL_DATABASE: matomo | |
| GEMINI_MODEL: ${{ github.event.inputs.gemini_model || 'gemini-2.5-flash' }} | |
| CODEX_MODEL: ${{ github.event.inputs.codex_model || 'gpt-5-mini' }} | |
| CLAUDE_MODEL: ${{ github.event.inputs.claude_model || 'sonnet' }} | |
| jobs: | |
| preflight: | |
| name: Preflight | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| should_run: ${{ steps.check.outputs.should_run }} | |
| has_gemini: ${{ steps.check.outputs.has_gemini }} | |
| has_codex: ${{ steps.check.outputs.has_codex }} | |
| has_claude: ${{ steps.check.outputs.has_claude }} | |
| reason: ${{ steps.check.outputs.reason }} | |
| steps: | |
| - id: check | |
| name: Check secret availability | |
| env: | |
| GEMINI_APIKEY: ${{ secrets.GEMINI_APIKEY }} | |
| OPENAI_APIKEY: ${{ secrets.OPENAI_APIKEY }} | |
| CLAUDE_APIKEY: ${{ secrets.CLAUDE_APIKEY }} | |
| run: | | |
| set -euo pipefail | |
| has_gemini=false | |
| has_codex=false | |
| has_claude=false | |
| if [ -n "${GEMINI_APIKEY:-}" ]; then | |
| has_gemini=true | |
| fi | |
| #if [ -n "${OPENAI_APIKEY:-}" ]; then | |
| # has_codex=true | |
| #fi | |
| #if [ -n "${CLAUDE_APIKEY:-}" ]; then | |
| # has_claude=true | |
| #fi | |
| echo "has_gemini=${has_gemini}" >> "$GITHUB_OUTPUT" | |
| echo "has_codex=${has_codex}" >> "$GITHUB_OUTPUT" | |
| echo "has_claude=${has_claude}" >> "$GITHUB_OUTPUT" | |
| if [ "$has_gemini" = false ] && [ "$has_codex" = false ] && [ "$has_claude" = false ]; then | |
| echo "should_run=false" >> "$GITHUB_OUTPUT" | |
| echo "reason=No provider secret is available in this context (likely fork PR)." >> "$GITHUB_OUTPUT" | |
| else | |
| echo "should_run=true" >> "$GITHUB_OUTPUT" | |
| echo "reason=Ready to run smoke checks for one or more providers." >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Preflight summary | |
| run: | | |
| { | |
| echo "## MCP AI Smoke Preflight" | |
| echo | |
| echo "- should_run: ${{ steps.check.outputs.should_run }}" | |
| echo "- Gemini available: ${{ steps.check.outputs.has_gemini }}" | |
| echo "- Codex available: ${{ steps.check.outputs.has_codex }}" | |
| echo "- Claude available: ${{ steps.check.outputs.has_claude }}" | |
| echo "- reason: ${{ steps.check.outputs.reason }}" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| smoke_gemini: | |
| name: Gemini smoke | |
| runs-on: ubuntu-24.04 | |
| needs: preflight | |
| if: needs.preflight.outputs.has_gemini == 'true' | |
| continue-on-error: true | |
| outputs: | |
| provider_outcome: ${{ steps.set_provider_outcome.outputs.provider_outcome }} | |
| timeout-minutes: 30 | |
| services: | |
| mysql: | |
| image: mysql:5.7 | |
| env: | |
| MYSQL_ROOT_PASSWORD: root | |
| MYSQL_DATABASE: matomo | |
| ports: | |
| - 3306:3306 | |
| options: >- | |
| --health-cmd="mysqladmin ping -h 127.0.0.1 -uroot -proot" | |
| --health-interval=10s | |
| --health-timeout=5s | |
| --health-retries=30 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 1 | |
| lfs: false | |
| persist-credentials: false | |
| - name: Setup shared smoke environment | |
| uses: ./.github/actions/setup-mcp-smoke-env | |
| with: | |
| plugin_name: ${{ env.PLUGIN_NAME }} | |
| matomo_dir: ${{ env.MATOMO_DIR }} | |
| base_url: ${{ env.BASE_URL }} | |
| mysql_host: ${{ env.MYSQL_HOST }} | |
| mysql_port: ${{ env.MYSQL_PORT }} | |
| mysql_user: ${{ env.MYSQL_USER }} | |
| mysql_password: ${{ env.MYSQL_PASSWORD }} | |
| mysql_database: ${{ env.MYSQL_DATABASE }} | |
| matomo_test_target: maximum_supported_matomo | |
| - name: Install Gemini CLI | |
| run: npm install -g @google/gemini-cli | |
| - name: Gemini MCP smoke suite | |
| id: smoke_suite | |
| continue-on-error: true | |
| run: ./.github/scripts/mcp-smoke/run-gemini-smoke.sh | |
| env: | |
| GEMINI_APIKEY: ${{ secrets.GEMINI_APIKEY }} | |
| GEMINI_MODEL: ${{ env.GEMINI_MODEL }} | |
| BASE_URL: ${{ env.BASE_URL }} | |
| STATE_FILE: ${{ github.workspace }}/.github/scripts/mcp-smoke/.state.json | |
| CASES_FILE: ${{ github.workspace }}/.github/scripts/mcp-smoke/cases.json | |
| PROMPTS_DIR: ${{ github.workspace }}/.github/scripts/mcp-smoke/prompts | |
| ARTIFACT_DIR: ${{ github.workspace }}/.github/scripts/mcp-smoke/artifacts | |
| MATOMO_LOG_FILE: ${{ env.MATOMO_DIR }}/tmp/logs/matomo.log | |
| PHP_SERVER_LOG_FILE: ${{ env.MATOMO_DIR }}/tmp/logs/php-server.log | |
| - name: Set Gemini provider outcome | |
| id: set_provider_outcome | |
| if: always() | |
| run: | | |
| provider_outcome="success" | |
| if [ "${{ job.status }}" != "success" ] || [ "${{ steps.smoke_suite.outcome }}" = "failure" ]; then | |
| provider_outcome="failure" | |
| elif [ "${{ steps.smoke_suite.outcome }}" = "cancelled" ]; then | |
| provider_outcome="cancelled" | |
| fi | |
| echo "provider_outcome=${provider_outcome}" >> "$GITHUB_OUTPUT" | |
| - name: Finalize Gemini smoke job | |
| if: always() | |
| uses: ./.github/actions/finalize-mcp-smoke-job | |
| with: | |
| provider_label: Gemini | |
| artifact_dir: ${{ github.workspace }}/.github/scripts/mcp-smoke/artifacts/gemini | |
| artifact_name: mcp-ai-smoke-gemini | |
| job_status: ${{ steps.set_provider_outcome.outputs.provider_outcome }} | |
| smoke_codex: | |
| name: Codex smoke | |
| runs-on: ubuntu-24.04 | |
| needs: preflight | |
| if: needs.preflight.outputs.has_codex == 'true' | |
| continue-on-error: true | |
| outputs: | |
| provider_outcome: ${{ steps.set_provider_outcome.outputs.provider_outcome }} | |
| timeout-minutes: 30 | |
| services: | |
| mysql: | |
| image: mysql:5.7 | |
| env: | |
| MYSQL_ROOT_PASSWORD: root | |
| MYSQL_DATABASE: matomo | |
| ports: | |
| - 3306:3306 | |
| options: >- | |
| --health-cmd="mysqladmin ping -h 127.0.0.1 -uroot -proot" | |
| --health-interval=10s | |
| --health-timeout=5s | |
| --health-retries=30 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 1 | |
| lfs: false | |
| persist-credentials: false | |
| - name: Setup shared smoke environment | |
| uses: ./.github/actions/setup-mcp-smoke-env | |
| with: | |
| plugin_name: ${{ env.PLUGIN_NAME }} | |
| matomo_dir: ${{ env.MATOMO_DIR }} | |
| base_url: ${{ env.BASE_URL }} | |
| mysql_host: ${{ env.MYSQL_HOST }} | |
| mysql_port: ${{ env.MYSQL_PORT }} | |
| mysql_user: ${{ env.MYSQL_USER }} | |
| mysql_password: ${{ env.MYSQL_PASSWORD }} | |
| mysql_database: ${{ env.MYSQL_DATABASE }} | |
| matomo_test_target: maximum_supported_matomo | |
| - name: Install Codex CLI | |
| run: npm install -g @openai/codex | |
| - name: Codex MCP smoke suite | |
| id: smoke_suite | |
| continue-on-error: true | |
| run: ./.github/scripts/mcp-smoke/run-codex-smoke.sh | |
| env: | |
| OPENAI_APIKEY: ${{ secrets.OPENAI_APIKEY }} | |
| CODEX_MODEL: ${{ env.CODEX_MODEL }} | |
| BASE_URL: ${{ env.BASE_URL }} | |
| STATE_FILE: ${{ github.workspace }}/.github/scripts/mcp-smoke/.state.json | |
| CASES_FILE: ${{ github.workspace }}/.github/scripts/mcp-smoke/cases.json | |
| PROMPTS_DIR: ${{ github.workspace }}/.github/scripts/mcp-smoke/prompts | |
| ARTIFACT_DIR: ${{ github.workspace }}/.github/scripts/mcp-smoke/artifacts | |
| MATOMO_LOG_FILE: ${{ env.MATOMO_DIR }}/tmp/logs/matomo.log | |
| PHP_SERVER_LOG_FILE: ${{ env.MATOMO_DIR }}/tmp/logs/php-server.log | |
| - name: Set Codex provider outcome | |
| id: set_provider_outcome | |
| if: always() | |
| run: | | |
| provider_outcome="success" | |
| if [ "${{ job.status }}" != "success" ] || [ "${{ steps.smoke_suite.outcome }}" = "failure" ]; then | |
| provider_outcome="failure" | |
| elif [ "${{ steps.smoke_suite.outcome }}" = "cancelled" ]; then | |
| provider_outcome="cancelled" | |
| fi | |
| echo "provider_outcome=${provider_outcome}" >> "$GITHUB_OUTPUT" | |
| - name: Finalize Codex smoke job | |
| if: always() | |
| uses: ./.github/actions/finalize-mcp-smoke-job | |
| with: | |
| provider_label: Codex | |
| artifact_dir: ${{ github.workspace }}/.github/scripts/mcp-smoke/artifacts/codex | |
| artifact_name: mcp-ai-smoke-codex | |
| job_status: ${{ steps.set_provider_outcome.outputs.provider_outcome }} | |
| smoke_claude: | |
| name: Claude smoke | |
| runs-on: ubuntu-24.04 | |
| needs: preflight | |
| if: needs.preflight.outputs.has_claude == 'true' | |
| continue-on-error: true | |
| outputs: | |
| provider_outcome: ${{ steps.set_provider_outcome.outputs.provider_outcome }} | |
| timeout-minutes: 30 | |
| services: | |
| mysql: | |
| image: mysql:5.7 | |
| env: | |
| MYSQL_ROOT_PASSWORD: root | |
| MYSQL_DATABASE: matomo | |
| ports: | |
| - 3306:3306 | |
| options: >- | |
| --health-cmd="mysqladmin ping -h 127.0.0.1 -uroot -proot" | |
| --health-interval=10s | |
| --health-timeout=5s | |
| --health-retries=30 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 1 | |
| lfs: false | |
| persist-credentials: false | |
| - name: Setup shared smoke environment | |
| uses: ./.github/actions/setup-mcp-smoke-env | |
| with: | |
| plugin_name: ${{ env.PLUGIN_NAME }} | |
| matomo_dir: ${{ env.MATOMO_DIR }} | |
| base_url: ${{ env.BASE_URL }} | |
| mysql_host: ${{ env.MYSQL_HOST }} | |
| mysql_port: ${{ env.MYSQL_PORT }} | |
| mysql_user: ${{ env.MYSQL_USER }} | |
| mysql_password: ${{ env.MYSQL_PASSWORD }} | |
| mysql_database: ${{ env.MYSQL_DATABASE }} | |
| matomo_test_target: maximum_supported_matomo | |
| - name: Install Claude CLI | |
| run: npm install -g @anthropic-ai/claude-code | |
| - name: Claude MCP smoke suite | |
| id: smoke_suite | |
| continue-on-error: true | |
| run: ./.github/scripts/mcp-smoke/run-claude-smoke.sh | |
| env: | |
| ANTHROPIC_API_KEY: ${{ secrets.CLAUDE_APIKEY }} | |
| CLAUDE_MODEL: ${{ env.CLAUDE_MODEL }} | |
| BASE_URL: ${{ env.BASE_URL }} | |
| STATE_FILE: ${{ github.workspace }}/.github/scripts/mcp-smoke/.state.json | |
| CASES_FILE: ${{ github.workspace }}/.github/scripts/mcp-smoke/cases.json | |
| PROMPTS_DIR: ${{ github.workspace }}/.github/scripts/mcp-smoke/prompts | |
| ARTIFACT_DIR: ${{ github.workspace }}/.github/scripts/mcp-smoke/artifacts | |
| MATOMO_LOG_FILE: ${{ env.MATOMO_DIR }}/tmp/logs/matomo.log | |
| PHP_SERVER_LOG_FILE: ${{ env.MATOMO_DIR }}/tmp/logs/php-server.log | |
| - name: Set Claude provider outcome | |
| id: set_provider_outcome | |
| if: always() | |
| run: | | |
| provider_outcome="success" | |
| if [ "${{ job.status }}" != "success" ] || [ "${{ steps.smoke_suite.outcome }}" = "failure" ]; then | |
| provider_outcome="failure" | |
| elif [ "${{ steps.smoke_suite.outcome }}" = "cancelled" ]; then | |
| provider_outcome="cancelled" | |
| fi | |
| echo "provider_outcome=${provider_outcome}" >> "$GITHUB_OUTPUT" | |
| - name: Finalize Claude smoke job | |
| if: always() | |
| uses: ./.github/actions/finalize-mcp-smoke-job | |
| with: | |
| provider_label: Claude | |
| artifact_dir: ${{ github.workspace }}/.github/scripts/mcp-smoke/artifacts/claude | |
| artifact_name: mcp-ai-smoke-claude | |
| job_status: ${{ steps.set_provider_outcome.outputs.provider_outcome }} | |
| summary: | |
| name: Final status | |
| runs-on: ubuntu-24.04 | |
| needs: [preflight, smoke_gemini, smoke_codex, smoke_claude] | |
| if: needs.preflight.outputs.should_run == 'true' | |
| steps: | |
| - name: Final status summary | |
| run: | | |
| { | |
| echo "## MCP AI Smoke Result" | |
| echo | |
| echo "- preflight: ${{ needs.preflight.outputs.should_run }}" | |
| echo "- Gemini available: ${{ needs.preflight.outputs.has_gemini }}" | |
| echo "- Codex available: ${{ needs.preflight.outputs.has_codex }}" | |
| echo "- Claude available: ${{ needs.preflight.outputs.has_claude }}" | |
| echo "- reason: ${{ needs.preflight.outputs.reason }}" | |
| echo "- Gemini provider outcome: ${{ needs.preflight.outputs.has_gemini == 'true' && needs.smoke_gemini.outputs.provider_outcome || 'skipped' }}" | |
| echo "- Codex provider outcome: ${{ needs.preflight.outputs.has_codex == 'true' && needs.smoke_codex.outputs.provider_outcome || 'skipped' }}" | |
| echo "- Claude provider outcome: ${{ needs.preflight.outputs.has_claude == 'true' && needs.smoke_claude.outputs.provider_outcome || 'skipped' }}" | |
| echo | |
| echo "Detailed per-case results remain in the individual provider jobs and uploaded artifacts." | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Evaluate overall result | |
| run: | | |
| set -euo pipefail | |
| gemini_result="${{ needs.preflight.outputs.has_gemini == 'true' && needs.smoke_gemini.outputs.provider_outcome || 'skipped' }}" | |
| codex_result="${{ needs.preflight.outputs.has_codex == 'true' && needs.smoke_codex.outputs.provider_outcome || 'skipped' }}" | |
| claude_result="${{ needs.preflight.outputs.has_claude == 'true' && needs.smoke_claude.outputs.provider_outcome || 'skipped' }}" | |
| ran_providers=0 | |
| failed_providers=0 | |
| for result in "$gemini_result" "$codex_result" "$claude_result"; do | |
| if [ "$result" != "skipped" ]; then | |
| ran_providers=$((ran_providers + 1)) | |
| fi | |
| if [ "$result" != "success" ] && [ "$result" != "skipped" ]; then | |
| failed_providers=$((failed_providers + 1)) | |
| fi | |
| done | |
| if [ "$ran_providers" -eq 0 ]; then | |
| echo "No provider jobs ran." | |
| exit 0 | |
| fi | |
| if [ "$failed_providers" -gt 0 ]; then | |
| echo "$failed_providers provider job(s) failed." | |
| exit 1 | |
| fi | |
| echo "All provider jobs succeeded." |