feat(trufflehog): centralize path exclusions and support .trufflehogignore

isaiah-grafana · isaiah-grafana · commit d2d989c919d6 · 2026-04-07T08:21:48.000-05:00
Add a centralized `trufflehog/exclude-paths.txt` for org-wide default
exclusions (vendor/, lock files, manifests, grafana.json, dashboards)
and support repo-local `.trufflehogignore` files for per-repo overrides.

Changes:
- Add trufflehog/exclude-paths.txt fetched at runtime from this repo
  (GitHub API → raw fallback → workflow ref fallback)
- Append repo-local .trufflehogignore to exclude patterns if present
- Simplify PR/merge-group scan loop with grep pre-filter
- Move ${{ }} expressions from run blocks into env blocks
- Fix jq CHANGELOG filter to use try/catch syntax
- Clean up org-required-trufflehog.yml comments
diff --git a/.github/workflows/org-required-trufflehog.yml b/.github/workflows/org-required-trufflehog.yml
@@ -21,10 +21,10 @@ permissions:
 jobs:
   secret-scan:
     name: TruffleHog Secret Scan
-    uses: grafana/security-github-actions/.github/workflows/reusable-trufflehog.yml@main
+    uses: grafana/security-github-actions/.github/workflows/reusable-trufflehog.yml@feature/trufflehog-global-excludes
     with:
-      # Fail on verified secrets - blocking mode
-      fail-on-verified: "true"      # Block on verified secrets
-      fail-on-unverified: "false"    # Don't block on unverified secrets
+      # Non-blocking: job succeeds; PR still gets comments/artifacts when findings exist
+      fail-on-verified: "false"      # Set "true" to fail on verified secrets
+      fail-on-unverified: "false"    # Set "true" to fail on unverified secrets
       runs-on: ${{ !github.event.repository.private && 'ubuntu-latest' || 'ubuntu-arm64-small' }}    # Use same runner pattern as zizmor
     secrets: inherit
diff --git a/.github/workflows/reusable-trufflehog.yml b/.github/workflows/reusable-trufflehog.yml
@@ -48,18 +48,63 @@ jobs:
         env:
           PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
           PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
-        run: git fetch --depth=1 origin "$PR_BASE_SHA" "$PR_HEAD_SHA"
+        run: git fetch --depth=1 origin "${PR_BASE_SHA}" "${PR_HEAD_SHA}"
 
       - name: Fetch base and head commits (merge_group)
         if: github.event_name == 'merge_group'
         env:
           MERGE_GROUP_BASE_SHA: ${{ github.event.merge_group.base_sha }}
           MERGE_GROUP_HEAD_SHA: ${{ github.event.merge_group.head_sha }}
-        run: git fetch --depth=1 origin "$MERGE_GROUP_BASE_SHA" "$MERGE_GROUP_HEAD_SHA"
+        run: git fetch --depth=1 origin "${MERGE_GROUP_BASE_SHA}" "${MERGE_GROUP_HEAD_SHA}"
 
       - name: Remove persisted credentials
         run: git config --unset-all http.https://github.com/.extraheader
 
+      - name: Fetch org-wide TruffleHog exclude patterns
+        env:
+          GH_TOKEN: ${{ github.token }}
+          WORKFLOW_REF: ${{ github.workflow_ref }}
+        run: |
+          DEST=/tmp/trufflehog-exclude.txt
+          REPO=grafana/security-github-actions
+          FILE_PATH=trufflehog/exclude-paths.txt
+
+          # Resolve the ref the calling workflow used (e.g. @main, @feature/branch, @v1).
+          CALLER_REF="main"
+          if [[ -n "${WORKFLOW_REF}" ]]; then
+            CALLER_REF=$(echo "${WORKFLOW_REF}" | sed 's|.*@||; s|^refs/heads/||; s|^refs/tags/||')
+          fi
+
+          LOADED=false
+          for REF in main "${CALLER_REF}"; do
+            [[ "$LOADED" == "true" ]] && break
+            if gh api "repos/${REPO}/contents/${FILE_PATH}?ref=${REF}" \
+                -H "Accept: application/vnd.github.v3.raw" -o "${DEST}" 2>/dev/null && [[ -s "${DEST}" ]]; then
+              echo "Loaded exclude patterns from ${REPO}@${REF} (GitHub API)"
+              LOADED=true
+            elif curl -fsSL "https://raw.githubusercontent.com/${REPO}/${REF}/${FILE_PATH}" \
+                -o "${DEST}" 2>/dev/null && [[ -s "${DEST}" ]]; then
+              echo "Loaded exclude patterns from raw.githubusercontent.com@${REF}"
+              LOADED=true
+            fi
+          done
+
+          if [[ "$LOADED" != "true" ]]; then
+            echo "::warning::Could not fetch TruffleHog exclude patterns from ${REPO} (tried main and ${CALLER_REF}). Scanning without exclusions."
+            touch "${DEST}"
+          fi
+
+          # Append repo-local .trufflehogignore if present (same Go regex format).
+          if [[ -f ".trufflehogignore" ]]; then
+            echo "" >> "${DEST}"
+            echo "# Repo-local .trufflehogignore" >> "${DEST}"
+            cat .trufflehogignore >> "${DEST}"
+            echo "Appended repo-local .trufflehogignore"
+          fi
+
+          echo "--- effective exclude patterns ---"
+          cat "${DEST}"
+
       - name: Install TruffleHog
         run: |
           # Download binary directly from GitHub releases for supply chain security
@@ -77,33 +122,10 @@ jobs:
           sudo chmod +x /usr/local/bin/trufflehog
           trufflehog --version
 
-      - name: Create global exclusions
-        run: |
-          # Create centralized exclusion patterns for common false positives
-          cat > /tmp/trufflehog-exclude.txt <<'EOF'
-          # Lock files and checksums (contain hashes, not secrets)
-          path:go\.sum$
-          path:go\.mod$
-          # Dependency manifests (contain URLs that trigger false positives)
-          path:package\.json$
-          path:package-lock\.json$
-          path:pnpm-lock\.yaml$
-          path:yarn\.lock$
-          path:poetry\.lock$
-          path:Pipfile\.lock$
-          path:uv\.lock$
-          path:Cargo\.lock$
-          path:Gemfile\.lock$
-          # Grafana plugin metadata
-          path:grafana\.json$
-          EOF
-
-          echo "Created global exclusion patterns:"
-          cat /tmp/trufflehog-exclude.txt
-
       - name: Scan for secrets
         id: scan
         env:
+          EVENT_NAME: ${{ github.event_name }}
           PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
           PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
           MERGE_GROUP_BASE_SHA: ${{ github.event.merge_group.base_sha }}
@@ -112,28 +134,24 @@ jobs:
           set +e
           echo "[]" > results.json
 
-          if [[ "${{ github.event_name }}" == "pull_request" ]] || [[ "${{ github.event_name }}" == "merge_group" ]]; then
-            # PR / merge queue: scan only paths that differ from base..head (not the entire checkout)
-            if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+          # Extract non-comment, non-blank patterns for the shell pre-filter.
+          grep -vE '^\s*#|^\s*$' /tmp/trufflehog-exclude.txt > /tmp/exclude-regexes.txt 2>/dev/null || true
+
+          if [[ "${EVENT_NAME}" == "pull_request" ]] || [[ "${EVENT_NAME}" == "merge_group" ]]; then
+            if [[ "${EVENT_NAME}" == "pull_request" ]]; then
               echo "Scanning changed files in PR..."
-              git diff --name-only "$PR_BASE_SHA" "$PR_HEAD_SHA" > changed-files.txt
+              git diff --name-only "${PR_BASE_SHA}" "${PR_HEAD_SHA}" > changed-files.txt
             else
               echo "Scanning changed files in merge group..."
-              git diff --name-only "$MERGE_GROUP_BASE_SHA" "$MERGE_GROUP_HEAD_SHA" > changed-files.txt
+              git diff --name-only "${MERGE_GROUP_BASE_SHA}" "${MERGE_GROUP_HEAD_SHA}" > changed-files.txt
             fi
 
             if [[ -s changed-files.txt ]]; then
               while IFS= read -r file; do
-                # Get just the filename
-                filename=$(basename "$file")
-
-                # Skip excluded files (use case statement for cleaner matching)
-                case "$filename" in
-                  go.sum|go.mod|package.json|package-lock.json|pnpm-lock.yaml|yarn.lock|poetry.lock|Pipfile.lock|uv.lock|Cargo.lock|Gemfile.lock|grafana.json)
-                    echo "Skipping: ${file} (excluded manifest/lock file)"
-                    continue
-                    ;;
-                esac
+                if [[ -s /tmp/exclude-regexes.txt ]] && echo "$file" | grep -qEf /tmp/exclude-regexes.txt 2>/dev/null; then
+                  echo "Skipping: ${file} (matches exclude pattern)"
+                  continue
+                fi
 
                 if [[ -f "${file}" ]]; then
                   echo "Scanning: ${file}"
@@ -144,7 +162,6 @@ jobs:
               echo "No files changed"
             fi
           else
-            # push to main (and any other events): full filesystem scan
             echo "Scanning current filesystem..."
             trufflehog filesystem . --exclude-paths /tmp/trufflehog-exclude.txt --concurrency 16 --json --no-update --results=verified,unverified > results.ndjson || true
           fi
@@ -161,8 +178,12 @@ jobs:
                 # Filter out CHANGELOG git hashes if we have results
                 if jq -e 'length > 0' results.json >/dev/null 2>&1; then
                   jq '[.[] | select(
-                    ((.SourceMetadata?.Data?.Filesystem?.file // .SourceMetadata?.Data?.Git?.file) // "") as $file |
-                    !(($file | test("CHANGELOG|HISTORY\\.md|NEWS\\.md"; "i")) and (.Raw | test("^[0-9a-f]{7,40}$"; "i")))
+                    (
+                      (try .SourceMetadata.Data.Filesystem.file catch null) //
+                      (try .SourceMetadata.Data.Git.file catch null) //
+                      ""
+                    ) as $file |
+                    ((($file | test("CHANGELOG|HISTORY\\.md|NEWS\\.md"; "i")) and (.Raw | test("^[0-9a-f]{7,40}$"; "i"))) | not)
                   )]' results.json > results.json.tmp && mv results.json.tmp results.json
                 fi
               else
@@ -196,26 +217,29 @@ jobs:
 
       - name: Delete resolved comment
         if: ${{ !cancelled() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && steps.scan.outputs.total == '0' }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GH_REPOSITORY: ${{ github.repository }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
         run: |
           # Find and delete TruffleHog comment if secrets have been resolved
-          COMMENT_ID=$(gh api repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments --jq '.[] | select(.body | contains("<!-- trufflehog-secret-scan-comment -->")) | .id' | head -1 || echo "")
+          COMMENT_ID=$(gh api "repos/${GH_REPOSITORY}/issues/${PR_NUMBER}/comments" --jq '.[] | select(.body | contains("<!-- trufflehog-secret-scan-comment -->")) | .id' | head -1 || echo "")
 
           if [[ -n "$COMMENT_ID" ]]; then
             echo "Secrets resolved - deleting previous warning comment (ID: ${COMMENT_ID})"
-            gh api -X DELETE /repos/${{ github.repository }}/issues/comments/${COMMENT_ID}
+            gh api -X DELETE "/repos/${GH_REPOSITORY}/issues/comments/${COMMENT_ID}"
             echo "Comment deleted successfully"
           else
             echo "No existing TruffleHog comment to delete"
           fi
-        env:
-          GH_TOKEN: ${{ github.token }}
 
       - name: Generate PR comment
         if: ${{ !cancelled() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && steps.scan.outputs.total > 0 }}
         id: comment-body
+        env:
+          VERIFIED: ${{ steps.scan.outputs.verified }}
+          UNVERIFIED: ${{ steps.scan.outputs.unverified }}
         run: |
-          VERIFIED=${{ steps.scan.outputs.verified }}
-          UNVERIFIED=${{ steps.scan.outputs.unverified }}
           TOTAL=$((VERIFIED+UNVERIFIED))
 
           if [[ $TOTAL -eq 0 ]]; then
@@ -232,9 +256,9 @@ jobs:
               "- " +
               (if .Verified then "**VERIFIED SECRET**" else "**Possible secret**" end) +
               " (" + .DetectorName + ") at `" +
-              ((.SourceMetadata?.Data?.Filesystem?.file // .SourceMetadata?.Data?.Git?.file) // "unknown") +
+              (((try .SourceMetadata.Data.Filesystem.file catch null) // (try .SourceMetadata.Data.Git.file catch null)) // "unknown") +
               ":" +
-              ((.SourceMetadata?.Data?.Filesystem?.line // .SourceMetadata?.Data?.Git?.line) | tostring) +
+              (((try .SourceMetadata.Data.Filesystem.line catch null) // (try .SourceMetadata.Data.Git.line catch null)) | tostring) +
               "` → `" +
               (if (.Raw | length) > 8 then (.Raw[:4] + "***" + .Raw[-4:]) else "***" end) +
               "`"' results.json 2>/dev/null || echo "- Error processing results")
@@ -281,23 +305,28 @@ jobs:
       - name: Create scan report
         env:
           GITHUB_REF_NAME: ${{ github.ref_name }}
+          GH_REPOSITORY: ${{ github.repository }}
+          GH_SHA: ${{ github.sha }}
+          TOTAL_SECRETS: ${{ steps.scan.outputs.total }}
+          VERIFIED_SECRETS: ${{ steps.scan.outputs.verified }}
+          UNVERIFIED_SECRETS: ${{ steps.scan.outputs.unverified }}
         run: |
           {
             echo "TruffleHog Scan Report"
             echo "====================="
             echo "Date: $(date)"
-            echo "Repository: ${{ github.repository }}"
+            echo "Repository: ${GH_REPOSITORY}"
             echo "Branch: ${GITHUB_REF_NAME}"
-            echo "Commit: ${{ github.sha }}"
+            echo "Commit: ${GH_SHA}"
             echo ""
             echo "Summary:"
-            echo "- Total secrets: ${{ steps.scan.outputs.total }}"
-            echo "- Verified: ${{ steps.scan.outputs.verified }}"
-            echo "- Unverified: ${{ steps.scan.outputs.unverified }}"
+            echo "- Total secrets: ${TOTAL_SECRETS}"
+            echo "- Verified: ${VERIFIED_SECRETS}"
+            echo "- Unverified: ${UNVERIFIED_SECRETS}"
             echo ""
             echo "Detailed Results:"
             if [[ -f "results.json" && -s "results.json" ]] && jq empty results.json 2>/dev/null; then
-              jq -r '.[] | "- " + (if .Verified then "VERIFIED" else "Unverified" end) + " " + .DetectorName + " at " + ((.SourceMetadata?.Data?.Filesystem?.file // .SourceMetadata?.Data?.Git?.file) // "unknown") + ":" + ((.SourceMetadata?.Data?.Filesystem?.line // .SourceMetadata?.Data?.Git?.line) | tostring) + " → " + (if (.Raw | length) > 8 then (.Raw[:4] + "***" + .Raw[-4:]) else "***" end)' results.json 2>/dev/null || echo "Error processing results"
+              jq -r '.[] | "- " + (if .Verified then "VERIFIED" else "Unverified" end) + " " + .DetectorName + " at " + (((try .SourceMetadata.Data.Filesystem.file catch null) // (try .SourceMetadata.Data.Git.file catch null)) // "unknown") + ":" + (((try .SourceMetadata.Data.Filesystem.line catch null) // (try .SourceMetadata.Data.Git.line catch null)) | tostring) + " → " + (if (.Raw | length) > 8 then (.Raw[:4] + "***" + .Raw[-4:]) else "***" end)' results.json 2>/dev/null || echo "Error processing results"
             else
               echo "No secrets detected"
             fi
diff --git a/trufflehog/exclude-paths.txt b/trufflehog/exclude-paths.txt
@@ -0,0 +1,31 @@
+# TruffleHog path exclusions — one Go regex per line.
+# Edit this file and merge to main. CI fetches it at runtime and passes
+# it directly to `trufflehog --exclude-paths`.
+#
+# Syntax: Go regexp (https://pkg.go.dev/regexp/syntax).
+# A file is excluded when ANY pattern matches its path.
+# Lines starting with # and blank lines are ignored.
+
+# Go vendor directory (third-party code, not repo secrets)
+vendor/
+
+# Lock files and checksums (contain hashes, not secrets)
+go\.sum$
+go\.mod$
+
+# Dependency manifests (contain URLs / hashes that trigger false positives)
+package\.json$
+package-lock\.json$
+pnpm-lock\.yaml$
+yarn\.lock$
+poetry\.lock$
+Pipfile\.lock$
+uv\.lock$
+Cargo\.lock$
+Gemfile\.lock$
+
+# Grafana plugin metadata
+grafana\.json$
+
+# Grafana dashboards (user-supplied site content, full of base64/hashes)
+content/grafana/dashboards
