Skip to content

Commit 133c651

Browse files
marc-casavantCopilot
andcommitted
WIP: misc updates to scripts for multi-server profiling tests
Co-authored-by: Copilot <copilot@github.com>
1 parent 6bba49b commit 133c651

7 files changed

Lines changed: 234 additions & 21 deletions

File tree

src/tests/multi-server/scripts/build_image.sh

Lines changed: 0 additions & 17 deletions
This file was deleted.

src/tests/multi-server/builds/Dockerfile.multi-server-prof renamed to src/tests/multi-server/scripts/docker/build/Dockerfile.multi-server-prof

File renamed without changes.
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#!/bin/bash
2+
for arg in "$@"; do
3+
case $arg in
4+
BUILD_PLATFORM=*) BUILD_PLATFORM="${arg#*=}" ;;
5+
esac
6+
done
7+
8+
# This allows us to build an image on Apple Silicon where the base image was built on an linux/amd64 platform.
9+
# Example usage: BUILD_PLATFORM=linux/amd64 ./build_image.sh
10+
PLATFORM_ARG=""
11+
if [ -n "${BUILD_PLATFORM}" ]; then
12+
PLATFORM_ARG="--platform=${BUILD_PLATFORM}"
13+
fi
14+
15+
docker build ${PLATFORM_ARG} -f src/tests/multi-server/scripts/docker/build/Dockerfile.multi-server-prof/Dockerfile.multi-server-prof -t freeradius-prof:latest .

src/tests/multi-server/scripts/run_container.sh renamed to src/tests/multi-server/scripts/docker/build/run_container.sh

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,11 @@ for arg in "$@"; do
55
esac
66
done
77

8+
# This allows us to run a container on Apple Silicon where the base image was built on an linux/amd64 platform.
9+
# Example usage: BUILD_PLATFORM=linux/amd64 ./run_container.sh
810
PLATFORM_ARG=""
9-
if [ "${BUILD_PLATFORM}" = "amd64" ]; then
10-
PLATFORM_ARG="--platform=linux/amd64"
11-
elif [ -n "${BUILD_PLATFORM}" ]; then
12-
PLATFORM_ARG="--platform=linux/${BUILD_PLATFORM}"
11+
if [ -n "${BUILD_PLATFORM}" ]; then
12+
PLATFORM_ARG="--platform=${BUILD_PLATFORM}"
1313
fi
1414

1515
docker run -it --rm ${PLATFORM_ARG} -v "$(pwd)/prof-results:/etc/prof-results" --name freeradius-radenv-container freeradius-prof:latest
Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
## generate_callgrind_report.py
2+
3+
python3 src/tests/multi-server/scripts/generate_callgrind_report.py \
4+
<results_dir> \
5+
--title "FreeRADIUS prof-accept 5min" \
6+
--text-output valgrind_report_radenv_prof_accept.txt \
7+
--md-output valgrind_report_radenv_prof_accept.md
8+
9+
## Generate text based report from Valgrind/Callgrind results
10+
callgrind_annotate $(find . -name "callgrind.out.*" -size +0c | sort) > callgrind_report.txt
11+
12+
## Generate SVG sharable file of valgrind/callgrind results
13+
14+
Dependency: ```brew install gprof2dot```
15+
16+
Generate SVG file for one worker thread:
17+
```
18+
gprof2dot --format=callgrind \
19+
<path-to-prof-results>/callgrind.out.1004-04 \
20+
| dot -Tsvg -o callgraph_thread04.svg
21+
```
22+
23+
Generate SVG file per worker thread:
24+
```
25+
for f in <path-to-prof-results>/callgrind.out.1004-{04..12}; do
26+
thread=$(grep "^thread:" "$f" | awk '{print $2}')
27+
gprof2dot --format=callgrind "$f" \
28+
| dot -Tsvg -o "callgraph_thread${thread}.svg"
29+
done
30+
```
Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
#!/usr/bin/env python3
2+
"""Generate text and markdown profiling reports from callgrind output files."""
3+
4+
import argparse
5+
import os
6+
import re
7+
import subprocess
8+
import sys
9+
from collections import defaultdict
10+
11+
12+
def parse_thread_number(callgrind_file):
13+
with open(callgrind_file) as f:
14+
for line in f:
15+
if line.startswith('thread:'):
16+
return int(line.split()[1])
17+
return 0
18+
19+
20+
def parse_summary_ir(callgrind_file):
21+
with open(callgrind_file) as f:
22+
for line in f:
23+
if line.startswith('summary:'):
24+
return int(line.split()[1])
25+
return 0
26+
27+
28+
def run_callgrind_annotate(callgrind_file):
29+
result = subprocess.run(
30+
['callgrind_annotate', '--auto=no', '--threshold=100', callgrind_file],
31+
capture_output=True, text=True
32+
)
33+
return result.stdout
34+
35+
36+
def parse_module_entries(annotate_output):
37+
"""Extract rlm_* and proto_load function rows from callgrind_annotate output."""
38+
entries = []
39+
for line in annotate_output.splitlines():
40+
if 'rlm_' not in line and 'proto_load' not in line:
41+
continue
42+
if not line.strip() or line.startswith('-') or line.startswith('='):
43+
continue
44+
45+
ir_match = re.match(r'^\s*([\d,]+)\s+\(\s*([\d.]+)%\)', line)
46+
if not ir_match:
47+
continue
48+
49+
func_match = re.search(r'([^/\s]+):(\w+)\s+\[([^\]]+)\]', line)
50+
if not func_match:
51+
continue
52+
53+
entries.append({
54+
'ir': int(ir_match.group(1).replace(',', '')),
55+
'ir_pct': float(ir_match.group(2)),
56+
'function': func_match.group(2),
57+
'lib': os.path.basename(func_match.group(3)),
58+
})
59+
return entries
60+
61+
62+
def fmt_ir(n):
63+
return f"{n:,}"
64+
65+
66+
def generate_markdown(results_dir, thread_data, title):
67+
lines = []
68+
lines.append(f"# {title}")
69+
lines.append("")
70+
lines.append(f"**Results:** `{results_dir}`")
71+
lines.append("")
72+
73+
# Collect all unique function+lib pairs
74+
lib_to_funcs = defaultdict(set)
75+
for td in thread_data.values():
76+
for e in td['entries']:
77+
lib_to_funcs[e['lib']].add(e['function'])
78+
79+
lines.append("## Functions Found")
80+
lines.append("")
81+
lines.append("| Function | Library |")
82+
lines.append("|---|---|")
83+
for lib in sorted(lib_to_funcs):
84+
funcs = ', '.join(f'`{f}`' for f in sorted(lib_to_funcs[lib]))
85+
lines.append(f"| {funcs} | `{lib}` |")
86+
lines.append("")
87+
88+
lines.append("## CPU Share (Ir = Instructions Retired)")
89+
lines.append("")
90+
91+
for thread_num, td in sorted(thread_data.items()):
92+
if not td['entries']:
93+
continue
94+
95+
total_ir = td['total_ir']
96+
lines.append(f"### Thread {thread_num:02d} — Total: {fmt_ir(total_ir)} Ir")
97+
lines.append("")
98+
lines.append("| Function | Library | Ir | % of Thread |")
99+
lines.append("|---|---|---|---|")
100+
101+
module_total = 0
102+
for e in sorted(td['entries'], key=lambda x: -x['ir']):
103+
lines.append(f"| `{e['function']}` | `{e['lib']}` | {fmt_ir(e['ir'])} | {e['ir_pct']:.2f}% |")
104+
module_total += e['ir']
105+
106+
module_pct = (module_total / total_ir * 100) if total_ir else 0
107+
lines.append(f"| **Total** | | **{fmt_ir(module_total)}** | **{module_pct:.2f}%** |")
108+
lines.append("")
109+
110+
all_module_ir = sum(e['ir'] for td in thread_data.values() for e in td['entries'])
111+
all_total_ir = sum(td['total_ir'] for td in thread_data.values())
112+
overall_pct = (all_module_ir / all_total_ir * 100) if all_total_ir else 0
113+
114+
lines.append("## Takeaway")
115+
lines.append("")
116+
lines.append(
117+
f"`rlm_*` and `proto_load` combined account for **{overall_pct:.2f}% of total instructions** "
118+
f"across all threads ({fmt_ir(all_module_ir)} of {fmt_ir(all_total_ir)} Ir total)."
119+
)
120+
lines.append("")
121+
122+
return "\n".join(lines)
123+
124+
125+
def main():
126+
parser = argparse.ArgumentParser(description="Generate profiling reports from callgrind results")
127+
parser.add_argument("results_dir", help="Directory containing callgrind.out.* files")
128+
parser.add_argument("--title", default=None, help="Report title")
129+
parser.add_argument("--text-output", default=None, help="Path for combined callgrind_annotate text report")
130+
parser.add_argument("--md-output", default=None, help="Path for markdown summary report")
131+
args = parser.parse_args()
132+
133+
results_dir = args.results_dir
134+
if not os.path.isdir(results_dir):
135+
print(f"error: {results_dir} is not a directory", file=sys.stderr)
136+
sys.exit(1)
137+
138+
files = sorted([
139+
os.path.join(results_dir, f)
140+
for f in os.listdir(results_dir)
141+
if re.match(r'callgrind\.out\.\d+(-\d+)?$', f) and os.path.getsize(os.path.join(results_dir, f)) > 0
142+
])
143+
144+
if not files:
145+
print(f"error: no callgrind.out.* files found in {results_dir}", file=sys.stderr)
146+
sys.exit(1)
147+
148+
title = args.title or f"FreeRADIUS Callgrind Profile: {os.path.basename(os.path.normpath(results_dir))}"
149+
150+
thread_data = {}
151+
text_sections = []
152+
153+
for f in files:
154+
thread_num = parse_thread_number(f)
155+
total_ir = parse_summary_ir(f)
156+
print(f" {os.path.basename(f)}: thread {thread_num:02d}, {total_ir:,} Ir", file=sys.stderr)
157+
158+
annotate_output = run_callgrind_annotate(f)
159+
text_sections.append(f"{'='*80}\n{os.path.basename(f)} (thread {thread_num:02d})\n{'='*80}\n{annotate_output}")
160+
161+
if thread_num not in thread_data:
162+
thread_data[thread_num] = {'total_ir': 0, 'entries': []}
163+
thread_data[thread_num]['total_ir'] += total_ir
164+
thread_data[thread_num]['entries'].extend(parse_module_entries(annotate_output))
165+
166+
if args.text_output:
167+
with open(args.text_output, 'w') as out:
168+
out.write("\n\n".join(text_sections))
169+
print(f"text report -> {args.text_output}", file=sys.stderr)
170+
171+
md = generate_markdown(results_dir, thread_data, title)
172+
173+
if args.md_output:
174+
with open(args.md_output, 'w') as out:
175+
out.write(md)
176+
print(f"markdown report -> {args.md_output}", file=sys.stderr)
177+
else:
178+
print(md)
179+
180+
181+
if __name__ == '__main__':
182+
main()

src/tests/multi-server/scripts/start_valgrind_profiling.sh renamed to src/tests/multi-server/scripts/profiling/start_valgrind_profiling.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ source /etc/freeradius/proto_load_config.env
77

88
# Calculate approximate send duration
99
SEND_DURATION=$(( TEST_LOADGEN_NUM_MESSAGES / TEST_LOADGEN_START_PPS ))
10+
PROFILE_DURATION_BUFFER=15
11+
PROFILE_DURATION=$SEND_DURATION-$PROFILE_DURATION_BUFFER
1012

1113
# Start freeradius under valgrind with instrumentation off
1214
valgrind \
@@ -37,3 +39,4 @@ sleep ${SEND_DURATION}
3739

3840
# Graceful shutdown (equivalent to Ctrl+C)
3941
kill -SIGINT ${FR_PID}
42+
wait

0 commit comments

Comments
 (0)