e2e.yaml

name: E2E Tests

on:
  schedule:
    - cron: "0 5 * * *"
  workflow_dispatch:
    inputs:
      debug_enabled:
        type: boolean
        description: "Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)"
        required: false
        default: false
      target_ref:
        type: string
        description: "Target ref to run the tests against"
        required: false
  pull_request_target:
    types:
      - opened
      - reopened
      - synchronize
    paths:
      - "**.go"
      - ".github/workflows/**"
      - "test/testdata/**"

jobs:
  e2e-tests:
    # Run on schedule, unconditional workflow_dispatch,
    # or pull_request_target if the actor has write/admin permissions.
    if: >
      github.event_name == 'schedule' ||
      github.event_name == 'workflow_dispatch' ||
      github.event_name == 'pull_request_target'
    concurrency:
      group: ${{ github.workflow }}-${{ matrix.provider }}-${{ github.event.pull_request.number || github.ref_name }}
      cancel-in-progress: true

    name: e2e tests
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        provider:
          [
            github_public,
            github_ghe,
            gitlab_bitbucket,
            gitea_1,
            gitea_2,
            gitea_3,
            concurrency,
          ]

    env:
      CONTROLLER_DOMAIN_URL: controller.paac-127-0-0-1.nip.io
      KOCACHE: /tmp/ko-cache
      KO_DOCKER_REPO: localhost:5000
      KUBECONFIG: /home/runner/.kube/config.kind
      TARGET_TEAM_SLUGS: "pipeline-as-code,pipeline-as-code-contributors"
      TEST_BITBUCKET_CLOUD_API_URL: https://api.bitbucket.org/2.0
      TEST_BITBUCKET_CLOUD_E2E_REPOSITORY: cboudjna/pac-e2e-tests
      TEST_BITBUCKET_CLOUD_TOKEN: ${{ secrets.BITBUCKET_CLOUD_TOKEN }}
      TEST_BITBUCKET_CLOUD_USER: cboudjna
      TEST_BITBUCKET_SERVER_API_URL: ${{ secrets.BITBUCKET_SERVER_API_URL }}
      TEST_BITBUCKET_SERVER_E2E_REPOSITORY: PAC/pac-e2e-tests
      TEST_BITBUCKET_SERVER_TOKEN: ${{ secrets.BITBUCKET_SERVER_TOKEN }}
      TEST_BITBUCKET_SERVER_USER: pipelines
      TEST_BITBUCKET_SERVER_WEBHOOK_SECRET: ${{ secrets.BITBUCKET_SERVER_WEBHOOK_SECRET }}
      TEST_EL_URL: http://controller.paac-127-0-0-1.nip.io
      TEST_EL_WEBHOOK_SECRET: ${{ secrets.WEBHOOK_SECRET }}
      TEST_GITEA_API_URL: http://localhost:3000
      TEST_GITEA_INTERNAL_URL: http://gitea.gitea:3000
      TEST_GITEA_PASSWORD: pac
      TEST_GITEA_REPO_OWNER: pac/pac
      TEST_GITEA_USERNAME: pac
      TEST_GITHUB_API_URL: api.github.com
      TEST_GITHUB_PRIVATE_TASK_NAME: task-remote
      TEST_GITHUB_PRIVATE_TASK_URL: https://github.com/openshift-pipelines/pipelines-as-code-e2e-tests-private/blob/main/remote_task.yaml
      TEST_GITHUB_REPO_INSTALLATION_ID: ${{ vars.INSTALLATION_ID }}
      TEST_GITHUB_REPO_OWNER_GITHUBAPP: openshift-pipelines/pipelines-as-code-e2e-tests
      TEST_GITHUB_REPO_OWNER_WEBHOOK: openshift-pipelines/pipelines-as-code-e2e-tests-webhook
      TEST_GITHUB_SECOND_API_URL: ghe.pipelinesascode.com
      TEST_GITHUB_SECOND_EL_URL: http://ghe.paac-127-0-0-1.nip.io
      TEST_GITHUB_SECOND_REPO_INSTALLATION_ID: 1
      TEST_GITHUB_SECOND_REPO_OWNER_GITHUBAPP: pipelines-as-code/e2e
      TEST_GITHUB_SECOND_TOKEN: ${{ secrets.TEST_GITHUB_SECOND_TOKEN }}
      TEST_GITHUB_TOKEN: ${{ secrets.GH_APPS_TOKEN }}
      TEST_GITLAB_PROJECT_ID: ${{ vars.TEST_GITLAB_PROJECT_ID }}
      TEST_GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }}
    steps:
      - uses: actions/checkout@v6
        with:
          ref: ${{ inputs.target_ref || github.event.pull_request.head.sha || github.sha }}

      # Step to check PR author's org membership and repo permissions.
      # This step will fail the job if checks do not pass, skipping subsequent steps.
      - name: Check user permissions on PRs
        if: github.event_name == 'pull_request_target'
        uses: actions/github-script@v8
        with:
          script: |
            if (!context || !context.payload || !context.payload.pull_request) {
              core.setFailed('Invalid GitHub context: missing required pull_request information');
              return;
            }

            async function run() {
              const actor = context.payload.pull_request.user.login;
              const repoOwner = context.repo.owner;
              const repoName = context.repo.repo;
              const targetOrg = context.repo.owner;

              core.info(`🔍 Starting permission check for user: @${actor}`);
              core.info(`📋 Repository: ${repoOwner}/${repoName}`);
              core.info(`🏢 Target organization: ${targetOrg}`);

              // Condition 1: Check if the user is a trusted bot.
              const trustedBots = ["dependabot[bot]", "renovate[bot]"];
              core.info(`🤖 Checking if @${actor} is a trusted bot...`);
              core.info(`   Trusted bots list: ${trustedBots.join(', ')}`);

              if (trustedBots.includes(actor)) {
                core.info(`✅ Condition met: User @${actor} is a trusted bot. Proceeding.`);
                return; // Success
              }
              core.info(`   ❌ User @${actor} is not a trusted bot.`);

              // Condition 2: Check for public membership in the target organization.
              core.info(`\n👥 Condition 2: Checking organization and team membership...`);
              core.info(
                `User @${actor} is not a trusted bot. Checking for membership in '${targetOrg}'...`,
              );
              try {
                // Optional: check membership in one or more org teams (set TARGET_TEAM_SLUGS as comma-separated slugs in workflow env)
                const teamSlugsEnv = process.env.TARGET_TEAM_SLUGS || "";
                const teamSlugs = teamSlugsEnv
                  .split(",")
                  .map((s) => s.trim())
                  .filter(Boolean);

                core.info(`🔧 TARGET_TEAM_SLUGS environment variable: "${teamSlugsEnv}"`);
                core.info(`📝 Parsed team slugs: [${teamSlugs.join(', ')}]`);

                if (teamSlugs.length > 0) {
                  core.info(`🔍 Checking team membership for ${teamSlugs.length} team(s)...`);
                  for (const team_slug of teamSlugs) {
                    core.info(`   Checking team: ${team_slug}...`);
                    try {
                      const membership = await github.rest.teams.getMembershipForUserInOrg({
                        org: targetOrg,
                        team_slug,
                        username: actor,
                      });
                      core.info(`   API response for team '${team_slug}': ${JSON.stringify(membership.data)}`);
                      if (
                        membership &&
                        membership.data &&
                        membership.data.state === "active"
                      ) {
                        core.info(
                          `✅ Condition met: User @${actor} is a member of team '${team_slug}' in '${targetOrg}'. Proceeding.`,
                        );
                        return; // Success
                      } else {
                        core.info(`   ⚠️ Team membership found but state is not 'active': ${membership.data.state}`);
                      }
                    } catch (err) {
                      // Not a member of this team or team doesn't exist — continue to next
                      core.info(
                        `   ❌ User @${actor} is not a member of team '${team_slug}' (or team not found). Error: ${err.message}`,
                      );
                    }
                  }
                  // If we tried team checks and none matched, continue to next org membership checks
                  core.info(
                    `ⓘ User @${actor} is not a member of any configured teams in '${targetOrg}'. Falling back to org membership checks.`,
                  );
                } else {
                  core.info(`ℹ️ No teams configured in TARGET_TEAM_SLUGS. Skipping team membership checks.`);
                }
                core.info(`🏢 Checking organization membership for @${actor} in '${targetOrg}'...`);
                try {
                  core.info(`   Attempting checkMembershipForUser API call...`);
                  await github.rest.orgs.checkMembershipForUser({
                    org: targetOrg,
                    username: actor,
                  });
                  core.info(
                    `✅ Condition met: User @${actor} is a member of '${targetOrg}'. Proceeding.`,
                  );
                  return; // Success
                } catch (err) {
                  // Try public membership as fallback
                  core.info(`   ❌ Private membership check failed: ${err.message}`);
                  core.info(`   Attempting checkPublicMembershipForUser API call...`);
                  try {
                    await github.rest.orgs.checkPublicMembershipForUser({
                      org: targetOrg,
                      username: actor,
                    });
                    core.info(
                      `✅ Condition met: User @${actor} is a public member of '${targetOrg}'. Proceeding.`,
                    );
                    return; // Success
                  } catch (publicErr) {
                    // Neither private nor public member - will be caught by outer catch
                    core.info(`   ❌ Public membership check failed: ${publicErr.message}`);
                    throw publicErr;
                  }
                }
              } catch (error) {
                // This is not a failure, just one unmet condition. Log and continue.
                core.info(
                  `ⓘ User @${actor} is not a public member of '${targetOrg}'. Checking repository permissions as a fallback.`,
                );
              }

              // Condition 3: Check for write/admin permission on the repository.
              core.info(`\n🔐 Condition 3: Checking repository collaborator permissions...`);
              try {
                core.info(`   Attempting getCollaboratorPermissionLevel API call...`);
                const response = await github.rest.repos.getCollaboratorPermissionLevel({
                  owner: repoOwner,
                  repo: repoName,
                  username: actor,
                });

                const permission = response.data.permission;
                core.info(`   User @${actor} has '${permission}' permission on ${repoOwner}/${repoName}`);

                if (permission === "admin" || permission === "write") {
                  core.info(
                    `✅ Condition met: User @${actor} has '${permission}' repository permission. Proceeding.`,
                  );
                  return; // Success
                } else {
                  // If we reach here, no conditions were met. This is the final failure.
                  core.info(`   ❌ Permission '${permission}' is insufficient (requires 'write' or 'admin')`);
                  core.setFailed(
                    `❌ Permission check failed. User @${actor} did not meet any required conditions (trusted bot, org member, or repo write access).`,
                  );
                  return;
                }
              } catch (error) {
                // This error means they are not even a collaborator.
                core.info(`   ❌ Collaborator permission check failed: ${error.message}`);
                core.setFailed(
                  `❌ Permission check failed. User @${actor} is not a collaborator on this repository and did not meet other conditions.`,
                );
                return;
              }
            }

            run().catch(err => {
              core.error(`💥 Unexpected error during permission check: ${err.message}`);
              core.error(`   Stack trace: ${err.stack}`);
              core.setFailed(`Unexpected error during permission check: ${err.message}`);
            });

      - uses: actions/setup-go@v6
        with:
          go-version-file: "go.mod"

      - name: Cache ko layer cache
        uses: actions/cache@v5
        with:
          path: /tmp/ko-cache
          key: ${{ runner.os }}-ko-${{ hashFiles('go.sum') }}
          restore-keys: |
            ${{ runner.os }}-ko-

      - uses: ko-build/setup-ko@v0.9

      # Start binary build in background as soon as possible so we can optimize
      # the run of e2e when it's start later on
      - name: Build binaries in parallel with cluster installation
        run: |
          nohup make allbinaries > /tmp/binary-build.log 2>&1 &
          echo $! > /tmp/binary-build.pid

      - name: Install gosmee
        uses: jaxxstorm/action-install-gh-release@v2.1.0
        with:
          repo: chmouel/gosmee

      - name: Install Snazy
        uses: jaxxstorm/action-install-gh-release@v2.1.0
        with:
          repo: chmouel/snazy

      - name: Run gosmee for main controller
        run: |
          nohup gosmee client --saveDir /tmp/gosmee-replay ${{ secrets.PYSMEE_URL }} "http://${CONTROLLER_DOMAIN_URL}" > /tmp/gosmee-main.log 2>&1 &

      - name: Generate unique gosmee URL for Gitea tests
        if: startsWith(matrix.provider, 'gitea') || matrix.provider == 'concurrency'
        id: gosmee-url
        run: |
          SMEE_URL=$(curl -s https://hook.pipelinesascode.com -o /dev/null -w '%{redirect_url}')
          echo "Generated unique smee URL: ${SMEE_URL}"
          echo "url=${SMEE_URL}" >> "$GITHUB_OUTPUT"
          echo "TEST_GITEA_SMEEURL=${SMEE_URL}" >> "$GITHUB_ENV"

      - name: Setup tmate session
        uses: mxschmitt/action-tmate@v3
        # add all environment so we can debug easily
        env:
          CONTROLLER_DOMAIN_URL: controller.paac-127-0-0-1.nip.io
          KOCACHE: /tmp/ko-cache
          KO_DOCKER_REPO: localhost:5000
          KUBECONFIG: /home/runner/.kube/config.kind
          TARGET_TEAM_SLUGS: "pipeline-as-code,pipeline-as-code-contributors"
          TEST_BITBUCKET_CLOUD_API_URL: https://api.bitbucket.org/2.0
          TEST_BITBUCKET_CLOUD_E2E_REPOSITORY: cboudjna/pac-e2e-tests
          TEST_BITBUCKET_CLOUD_TOKEN: ${{ secrets.BITBUCKET_CLOUD_TOKEN }}
          TEST_BITBUCKET_CLOUD_USER: cboudjna
          TEST_BITBUCKET_SERVER_API_URL: ${{ secrets.BITBUCKET_SERVER_API_URL }}
          TEST_BITBUCKET_SERVER_E2E_REPOSITORY: PAC/pac-e2e-tests
          TEST_BITBUCKET_SERVER_TOKEN: ${{ secrets.BITBUCKET_SERVER_TOKEN }}
          TEST_BITBUCKET_SERVER_USER: pipelines
          TEST_BITBUCKET_SERVER_WEBHOOK_SECRET: ${{ secrets.BITBUCKET_SERVER_WEBHOOK_SECRET }}
          TEST_EL_URL: http://controller.paac-127-0-0-1.nip.io
          TEST_EL_WEBHOOK_SECRET: ${{ secrets.WEBHOOK_SECRET }}
          TEST_GITEA_API_URL: http://localhost:3000
          TEST_GITEA_PASSWORD: pac
          TEST_GITEA_REPO_OWNER: pac/pac
          TEST_GITEA_USERNAME: pac
          TEST_GITHUB_API_URL: api.github.com
          TEST_GITHUB_PRIVATE_TASK_NAME: task-remote
          TEST_GITHUB_PRIVATE_TASK_URL: https://github.com/openshift-pipelines/pipelines-as-code-e2e-tests-private/blob/main/remote_task.yaml
          TEST_GITHUB_REPO_INSTALLATION_ID: ${{ vars.INSTALLATION_ID }}
          TEST_GITHUB_REPO_OWNER_GITHUBAPP: openshift-pipelines/pipelines-as-code-e2e-tests
          TEST_GITHUB_REPO_OWNER_WEBHOOK: openshift-pipelines/pipelines-as-code-e2e-tests-webhook
          TEST_GITHUB_SECOND_API_URL: ghe.pipelinesascode.com
          TEST_GITHUB_SECOND_APPLICATION_ID: ${{ vars.TEST_GITHUB_SECOND_APPLICATION_ID }}
          TEST_GITHUB_SECOND_EL_URL: http://ghe.paac-127-0-0-1.nip.io
          TEST_GITHUB_SECOND_REPO_INSTALLATION_ID: 1
          TEST_GITHUB_SECOND_REPO_OWNER_GITHUBAPP: pipelines-as-code/e2e
          TEST_GITHUB_SECOND_TOKEN: ${{ secrets.TEST_GITHUB_SECOND_TOKEN }}
          TEST_GITHUB_SECOND_SMEE_URL: ${{ secrets.TEST_GITHUB_SECOND_SMEE_URL }}
          TEST_GITHUB_SECOND_WEBHOOK_SECRET: ${{ secrets.TEST_GITHUB_SECOND_WEBHOOK_SECRET }}
          TEST_GITHUB_TOKEN: ${{ secrets.GH_APPS_TOKEN }}
          TEST_GITLAB_API_URL: https://gitlab.com
          TEST_GITLAB_PROJECT_ID: ${{ vars.TEST_GITLAB_PROJECT_ID }}
          TEST_GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }}
          TEST_PROVIDER: ${{ matrix.provider }}
        if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
        with:
          detached: true
          limit-access-to-actor: true

      - name: Start installing cluster
        run: |
          export PAC_DIR=${PWD}
          bash -x ./hack/dev/kind/install.sh

      - name: Create PAC github-app-secret
        env:
          PAC_GITHUB_PRIVATE_KEY: ${{ secrets.APP_PRIVATE_KEY }}
          PAC_GITHUB_APPLICATION_ID: ${{ vars.APPLICATION_ID }}
          PAC_WEBHOOK_SECRET: ${{ secrets.WEBHOOK_SECRET }}
        run: |
          ./hack/gh-workflow-ci.sh create_pac_github_app_secret

      - name: Create second Github APP Controller on GHE
        env:
          TEST_GITHUB_SECOND_SMEE_URL: ${{ secrets.TEST_GITHUB_SECOND_SMEE_URL }}
          TEST_GITHUB_SECOND_PRIVATE_KEY: ${{ secrets.TEST_GITHUB_SECOND_PRIVATE_KEY }}
          TEST_GITHUB_SECOND_WEBHOOK_SECRET: ${{ secrets.TEST_GITHUB_SECOND_WEBHOOK_SECRET }}
          TEST_GITHUB_SECOND_APPLICATION_ID: ${{ vars.TEST_GITHUB_SECOND_APPLICATION_ID }}
        run: |
          ./hack/gh-workflow-ci.sh create_second_github_app_controller_on_ghe

      - name: Enable debug logging for e2e
        run: |
          set -euo pipefail
          kubectl -n pipelines-as-code patch configmap pac-config-logging --type merge -p '{"data":{"loglevel.pipelinesascode":"debug","loglevel.pac-watcher":"debug","loglevel.pipelines-as-code-webhook":"debug"}}'
          kubectl -n pipelines-as-code rollout restart deployment/pipelines-as-code-controller deployment/pipelines-as-code-webhook deployment/pipelines-as-code-watcher
          for name in controller webhook watcher; do
            echo "=== Waiting for $name to be ready ==="
            kubectl -n pipelines-as-code rollout status deployment/pipelines-as-code-$name --timeout=120s
          done

      # Adjusted step-level conditions based on the new job-level logic
      - name: Run E2E Tests
        # This step runs for schedule, PR target (if job started), or workflow_dispatch (if job started)
        # Remove the old label check which is no longer relevant for triggering.
        if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request_target' }}
        env:
          TEST_PROVIDER: ${{ matrix.provider }}
          TEST_BITBUCKET_CLOUD_TOKEN: ${{ secrets.BITBUCKET_CLOUD_TOKEN }}
          TEST_EL_WEBHOOK_SECRET: ${{ secrets.WEBHOOK_SECRET }}
          TEST_GITHUB_REPO_INSTALLATION_ID: ${{ vars.INSTALLATION_ID }}
          TEST_GITHUB_TOKEN: ${{ secrets.GH_APPS_TOKEN }}
          TEST_GITHUB_SECOND_TOKEN: ${{ secrets.TEST_GITHUB_SECOND_TOKEN }}
          TEST_GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }}
          TEST_BITBUCKET_SERVER_TOKEN: ${{ secrets.BITBUCKET_SERVER_TOKEN }}
          TEST_BITBUCKET_SERVER_API_URL: ${{ secrets.BITBUCKET_SERVER_API_URL }}
          TEST_BITBUCKET_SERVER_WEBHOOK_SECRET: ${{ secrets.BITBUCKET_SERVER_WEBHOOK_SECRET }}
        run: |
          ./hack/gh-workflow-ci.sh run_e2e_tests

      - name: Run E2E Tests on nightly
        # This step still runs specifically for schedule or workflow_dispatch
        if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
        env:
          NIGHTLY_E2E_TEST: "true"
          TEST_PROVIDER: ${{ matrix.provider }}
          TEST_BITBUCKET_CLOUD_TOKEN: ${{ secrets.BITBUCKET_CLOUD_TOKEN }}
          TEST_EL_WEBHOOK_SECRET: ${{ secrets.WEBHOOK_SECRET }}
          TEST_GITHUB_REPO_INSTALLATION_ID: ${{ vars.INSTALLATION_ID }}
          TEST_GITHUB_TOKEN: ${{ secrets.GH_APPS_TOKEN }}
          TEST_GITHUB_SECOND_TOKEN: ${{ secrets.TEST_GITHUB_SECOND_TOKEN }}
          TEST_GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }}
          TEST_BITBUCKET_SERVER_TOKEN: ${{ secrets.BITBUCKET_SERVER_TOKEN }}
          TEST_BITBUCKET_SERVER_API_URL: ${{ secrets.BITBUCKET_SERVER_API_URL }}
          TEST_BITBUCKET_SERVER_WEBHOOK_SECRET: ${{ secrets.BITBUCKET_SERVER_WEBHOOK_SECRET }}
        run: |
          ./hack/gh-workflow-ci.sh run_e2e_tests

      - name: Collect logs
        if: ${{ always() }}
        env:
          TEST_GITHUB_SECOND_SMEE_URL: ${{ secrets.TEST_GITHUB_SECOND_SMEE_URL }}
        run: |
          ./hack/gh-workflow-ci.sh collect_logs

      - name: Show controllers/watcher errors with Snazy
        if: ${{ always() }}
        run: |
          ./hack/gh-workflow-ci.sh output_logs

      - name: Upload artifacts
        if: ${{ always() }}
        uses: actions/upload-artifact@v6
        with:
          name: logs-e2e-tests-${{ matrix.provider }}
          path: /tmp/logs

  notify-slack:
    name: Notify Slack on Failures
    runs-on: ubuntu-latest
    needs: [e2e-tests]
    if: ${{ always() && github.ref_name == 'main' && github.event_name == 'schedule' }}
    steps:
      - uses: actions/checkout@v6
      - name: Download all artifacts
        uses: actions/download-artifact@v7
        with:
          path: artifacts
          pattern: logs-e2e-tests-*

      - name: Send Slack notification
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
        run: |
          ./hack/notify-slack.sh artifacts