Skip to content

Commit f2f56c5

Browse files
perf(trufflehog): batch PR changed-path scans
Feed filtered paths through GNU xargs -0 so a typical PR runs one TruffleHog process (avoids per-file startup) while argv stays under OS limits on very large diffs. Paths are still filtered for excludes and missing files (e.g. deletions in the diff).
1 parent 07958e8 commit f2f56c5

1 file changed

Lines changed: 18 additions & 4 deletions

File tree

.github/workflows/reusable-trufflehog.yml

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -147,17 +147,31 @@ jobs:
147147
fi
148148
149149
if [[ -s changed-files.txt ]]; then
150+
# One TruffleHog per argv batch (not per file): avoids repeated process startup on
151+
# large diffs; GNU xargs -0 splits when argv would exceed OS limits.
152+
paths=()
150153
while IFS= read -r file; do
151154
if [[ -s /tmp/exclude-regexes.txt ]] && echo "$file" | grep -qEf /tmp/exclude-regexes.txt 2>/dev/null; then
152155
echo "Skipping: ${file} (matches exclude pattern)"
153156
continue
154157
fi
155-
156158
if [[ -f "${file}" ]]; then
157-
echo "Scanning: ${file}"
158-
trufflehog filesystem "${file}" --exclude-paths /tmp/trufflehog-exclude.txt --concurrency 16 --json --no-update --results=verified,unverified >> results.ndjson || true
159+
paths+=("${file}")
159160
fi
160161
done < changed-files.txt
162+
163+
if ((${#paths[@]} > 0)); then
164+
echo "TruffleHog: ${#paths[@]} path(s), batched by xargs as needed"
165+
: > results.ndjson
166+
printf '%s\0' "${paths[@]}" | xargs -0 -r trufflehog filesystem \
167+
--exclude-paths /tmp/trufflehog-exclude.txt \
168+
--concurrency 16 \
169+
--json \
170+
--no-update \
171+
--results=verified,unverified >> results.ndjson || true
172+
else
173+
echo "No files to scan after excludes (only deletions or excluded paths)"
174+
fi
161175
else
162176
echo "No files changed"
163177
fi
@@ -379,7 +393,7 @@ jobs:
379393
id-token: write
380394
steps:
381395
- name: Get Prometheus secrets from Vault
382-
uses: grafana/shared-workflows/actions/get-vault-secrets@f1614b210386ac420af6807a997ac7f6d96e477a # get-vault-secrets/v1.3.1
396+
uses: grafana/shared-workflows/actions/get-vault-secrets@078c4a8af09e06d646077550f9e0f68171d5881e # get-vault-secrets/v1.3.1
383397
with:
384398
common_secrets: |
385399
PROMETHEUS_URL=grafana-bench:prometheus_url

0 commit comments

Comments
 (0)