benchmark-runner.sh raw
1 #!/bin/sh
2
3 # Benchmark runner script for testing multiple Nostr relay implementations
4 # This script coordinates testing all relays and aggregates results
5
6 set -e
7
8 # Configuration from environment variables
9 BENCHMARK_EVENTS="${BENCHMARK_EVENTS:-10000}"
10 BENCHMARK_WORKERS="${BENCHMARK_WORKERS:-8}"
11 BENCHMARK_DURATION="${BENCHMARK_DURATION:-60s}"
12 BENCHMARK_TARGETS="${BENCHMARK_TARGETS:-next-orly:8080,khatru-sqlite:3334,khatru-badger:3334,relayer-basic:7447,strfry:8080,nostr-rs-relay:8080}"
13 OUTPUT_DIR="${OUTPUT_DIR:-/reports}"
14
15 # Create output directory
16 mkdir -p "${OUTPUT_DIR}"
17
18 # Generate timestamp for this benchmark run
19 TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
20 RUN_DIR="${OUTPUT_DIR}/run_${TIMESTAMP}"
21 mkdir -p "${RUN_DIR}"
22
23 echo "=================================================="
24 echo "Nostr Relay Benchmark Suite"
25 echo "=================================================="
26 echo "Timestamp: $(date)"
27 echo "Events per test: ${BENCHMARK_EVENTS}"
28 echo "Concurrent workers: ${BENCHMARK_WORKERS}"
29 echo "Test duration: ${BENCHMARK_DURATION}"
30 echo "Graph traversal: ${BENCHMARK_GRAPH_TRAVERSAL:-false}"
31 echo "Output directory: ${RUN_DIR}"
32 echo "=================================================="
33
34 # Function to wait for relay to be ready
35 wait_for_relay() {
36 local name="$1"
37 local url="$2"
38 local max_attempts=60
39 local attempt=0
40
41 echo "Waiting for ${name} to be ready at ${url}..."
42
43 while [ $attempt -lt $max_attempts ]; do
44 # Try wget first to obtain an HTTP status code
45 local status=""
46 status=$(wget --quiet --server-response --tries=1 --timeout=5 "http://${url}" 2>&1 | awk '/^ HTTP\//{print $2; exit}')
47
48 # Fallback to curl to obtain an HTTP status code
49 if [ -z "$status" ]; then
50 status=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout 5 --max-time 5 "http://${url}" || echo 000)
51 fi
52
53 case "$status" in
54 101|200|400|404|426)
55 echo "${name} is ready! (HTTP ${status})"
56 return 0
57 ;;
58 esac
59
60 attempt=$((attempt + 1))
61 echo " Attempt ${attempt}/${max_attempts}: ${name} not ready yet (HTTP ${status:-none})..."
62 sleep 2
63 done
64
65 echo "ERROR: ${name} failed to become ready after ${max_attempts} attempts"
66 return 1
67 }
68
69 # Function to run benchmark against a specific relay
70 run_benchmark() {
71 local relay_name="$1"
72 local relay_url="$2"
73 local output_file="$3"
74
75 echo ""
76 echo "=================================================="
77 echo "Testing ${relay_name} at ws://${relay_url}"
78 echo "=================================================="
79
80 # Wait for relay to be ready
81 if ! wait_for_relay "${relay_name}" "${relay_url}"; then
82 echo "ERROR: ${relay_name} is not responding, skipping..."
83 echo "RELAY: ${relay_name}" > "${output_file}"
84 echo "STATUS: FAILED - Relay not responding" >> "${output_file}"
85 echo "ERROR: Connection failed" >> "${output_file}"
86 return 1
87 fi
88
89 # Run the benchmark
90 echo "Running benchmark against ${relay_name}..."
91
92 # Create temporary directory for this relay's data
93 TEMP_DATA_DIR="/tmp/benchmark_${relay_name}_$$"
94 mkdir -p "${TEMP_DATA_DIR}"
95
96 # Run benchmark and capture both stdout and stderr
97 if /app/benchmark \
98 -datadir="${TEMP_DATA_DIR}" \
99 -events="${BENCHMARK_EVENTS}" \
100 -workers="${BENCHMARK_WORKERS}" \
101 -duration="${BENCHMARK_DURATION}" \
102 > "${output_file}" 2>&1; then
103
104 echo "✓ Benchmark completed successfully for ${relay_name}"
105
106 # Add relay identification to the report
107 echo "" >> "${output_file}"
108 echo "RELAY_NAME: ${relay_name}" >> "${output_file}"
109 echo "RELAY_URL: ws://${relay_url}" >> "${output_file}"
110 echo "TEST_TIMESTAMP: $(date -Iseconds)" >> "${output_file}"
111 echo "BENCHMARK_CONFIG:" >> "${output_file}"
112 echo " Events: ${BENCHMARK_EVENTS}" >> "${output_file}"
113 echo " Workers: ${BENCHMARK_WORKERS}" >> "${output_file}"
114 echo " Duration: ${BENCHMARK_DURATION}" >> "${output_file}"
115
116 else
117 echo "✗ Benchmark failed for ${relay_name}"
118 echo "" >> "${output_file}"
119 echo "RELAY_NAME: ${relay_name}" >> "${output_file}"
120 echo "RELAY_URL: ws://${relay_url}" >> "${output_file}"
121 echo "STATUS: FAILED" >> "${output_file}"
122 echo "TEST_TIMESTAMP: $(date -Iseconds)" >> "${output_file}"
123 fi
124
125 # Clean up temporary data
126 rm -rf "${TEMP_DATA_DIR}"
127 }
128
129 # Function to run network graph traversal benchmark against a specific relay
130 run_graph_traversal_benchmark() {
131 local relay_name="$1"
132 local relay_url="$2"
133 local output_file="$3"
134
135 echo ""
136 echo "=================================================="
137 echo "Graph Traversal Benchmark: ${relay_name} at ws://${relay_url}"
138 echo "=================================================="
139
140 # Wait for relay to be ready
141 if ! wait_for_relay "${relay_name}" "${relay_url}"; then
142 echo "ERROR: ${relay_name} is not responding, skipping graph traversal..."
143 echo "RELAY: ${relay_name}" > "${output_file}"
144 echo "STATUS: FAILED - Relay not responding" >> "${output_file}"
145 echo "ERROR: Connection failed" >> "${output_file}"
146 return 1
147 fi
148
149 # Run the network graph traversal benchmark
150 echo "Running network graph traversal benchmark against ${relay_name}..."
151
152 # Create temporary directory for this relay's data
153 TEMP_DATA_DIR="/tmp/graph_benchmark_${relay_name}_$$"
154 mkdir -p "${TEMP_DATA_DIR}"
155
156 # Run graph traversal benchmark via WebSocket
157 if /app/benchmark \
158 -graph-network \
159 -relay-url="ws://${relay_url}" \
160 -datadir="${TEMP_DATA_DIR}" \
161 -workers="${BENCHMARK_WORKERS}" \
162 > "${output_file}" 2>&1; then
163
164 echo "✓ Graph traversal benchmark completed successfully for ${relay_name}"
165
166 # Add relay identification to the report
167 echo "" >> "${output_file}"
168 echo "RELAY_NAME: ${relay_name}" >> "${output_file}"
169 echo "RELAY_URL: ws://${relay_url}" >> "${output_file}"
170 echo "TEST_TYPE: Graph Traversal (100k pubkeys, 3-degree follows)" >> "${output_file}"
171 echo "TEST_TIMESTAMP: $(date -Iseconds)" >> "${output_file}"
172 echo "BENCHMARK_CONFIG:" >> "${output_file}"
173 echo " Workers: ${BENCHMARK_WORKERS}" >> "${output_file}"
174
175 else
176 echo "✗ Graph traversal benchmark failed for ${relay_name}"
177 echo "" >> "${output_file}"
178 echo "RELAY_NAME: ${relay_name}" >> "${output_file}"
179 echo "RELAY_URL: ws://${relay_url}" >> "${output_file}"
180 echo "TEST_TYPE: Graph Traversal" >> "${output_file}"
181 echo "STATUS: FAILED" >> "${output_file}"
182 echo "TEST_TIMESTAMP: $(date -Iseconds)" >> "${output_file}"
183 fi
184
185 # Clean up temporary data
186 rm -rf "${TEMP_DATA_DIR}"
187 }
188
189 # Function to generate aggregate report
190 generate_aggregate_report() {
191 local aggregate_file="${RUN_DIR}/aggregate_report.txt"
192
193 echo "Generating aggregate report..."
194
195 cat > "${aggregate_file}" << EOF
196 ================================================================
197 NOSTR RELAY BENCHMARK AGGREGATE REPORT
198 ================================================================
199 Generated: $(date -Iseconds)
200 Benchmark Configuration:
201 Events per test: ${BENCHMARK_EVENTS}
202 Concurrent workers: ${BENCHMARK_WORKERS}
203 Test duration: ${BENCHMARK_DURATION}
204
205 Relays tested: $(echo "${BENCHMARK_TARGETS}" | tr ',' '\n' | wc -l)
206
207 ================================================================
208 SUMMARY BY RELAY
209 ================================================================
210
211 EOF
212
213 # Process each relay's results
214 echo "${BENCHMARK_TARGETS}" | tr ',' '\n' | while IFS=':' read -r relay_name relay_port; do
215 if [ -z "${relay_name}" ] || [ -z "${relay_port}" ]; then
216 continue
217 fi
218
219 relay_file="${RUN_DIR}/${relay_name}_results.txt"
220
221 echo "Relay: ${relay_name}" >> "${aggregate_file}"
222 echo "----------------------------------------" >> "${aggregate_file}"
223
224 if [ -f "${relay_file}" ]; then
225 # Extract key metrics from the relay's report
226 if grep -q "STATUS: FAILED" "${relay_file}"; then
227 echo "Status: FAILED" >> "${aggregate_file}"
228 grep "ERROR:" "${relay_file}" | head -1 >> "${aggregate_file}" || echo "Error: Unknown failure" >> "${aggregate_file}"
229 else
230 echo "Status: COMPLETED" >> "${aggregate_file}"
231
232 # Extract performance metrics
233 grep "Events/sec:" "${relay_file}" | head -3 >> "${aggregate_file}" || true
234 grep "Success Rate:" "${relay_file}" | head -3 >> "${aggregate_file}" || true
235 grep "Avg Latency:" "${relay_file}" | head -3 >> "${aggregate_file}" || true
236 grep "P95 Latency:" "${relay_file}" | head -3 >> "${aggregate_file}" || true
237 grep "Memory:" "${relay_file}" | head -3 >> "${aggregate_file}" || true
238 fi
239 else
240 echo "Status: NO RESULTS FILE" >> "${aggregate_file}"
241 echo "Error: Results file not found" >> "${aggregate_file}"
242 fi
243
244 echo "" >> "${aggregate_file}"
245 done
246
247 cat >> "${aggregate_file}" << EOF
248
249 ================================================================
250 DETAILED RESULTS
251 ================================================================
252
253 Individual relay reports are available in:
254 $(ls "${RUN_DIR}"/*_results.txt 2>/dev/null | sed 's|^| - |' || echo " No individual reports found")
255
256 ================================================================
257 BENCHMARK COMPARISON TABLE
258 ================================================================
259
260 EOF
261
262 # Create a comparison table
263 printf "%-20s %-10s %-15s %-15s %-15s\n" "Relay" "Status" "Peak Tput/s" "Avg Latency" "Success Rate" >> "${aggregate_file}"
264 printf "%-20s %-10s %-15s %-15s %-15s\n" "----" "------" "-----------" "-----------" "------------" >> "${aggregate_file}"
265
266 echo "${BENCHMARK_TARGETS}" | tr ',' '\n' | while IFS=':' read -r relay_name relay_port; do
267 if [ -z "${relay_name}" ] || [ -z "${relay_port}" ]; then
268 continue
269 fi
270
271 relay_file="${RUN_DIR}/${relay_name}_results.txt"
272
273 if [ -f "${relay_file}" ]; then
274 if grep -q "STATUS: FAILED" "${relay_file}"; then
275 printf "%-20s %-10s %-15s %-15s %-15s\n" "${relay_name}" "FAILED" "-" "-" "-" >> "${aggregate_file}"
276 else
277 # Extract metrics for the table
278 peak_tput=$(grep "Events/sec:" "${relay_file}" | head -1 | awk '{print $2}' || echo "-")
279 avg_latency=$(grep "Avg Latency:" "${relay_file}" | head -1 | awk '{print $3}' || echo "-")
280 success_rate=$(grep "Success Rate:" "${relay_file}" | head -1 | awk '{print $3}' || echo "-")
281
282 printf "%-20s %-10s %-15s %-15s %-15s\n" "${relay_name}" "OK" "${peak_tput}" "${avg_latency}" "${success_rate}" >> "${aggregate_file}"
283 fi
284 else
285 printf "%-20s %-10s %-15s %-15s %-15s\n" "${relay_name}" "NO DATA" "-" "-" "-" >> "${aggregate_file}"
286 fi
287 done
288
289 echo "" >> "${aggregate_file}"
290 echo "================================================================" >> "${aggregate_file}"
291 echo "End of Report" >> "${aggregate_file}"
292 echo "================================================================" >> "${aggregate_file}"
293 }
294
295 # Main execution
296 echo "Starting relay benchmark suite..."
297
298 # Check if graph traversal mode is enabled
299 BENCHMARK_GRAPH_TRAVERSAL="${BENCHMARK_GRAPH_TRAVERSAL:-false}"
300
301 # Parse targets and run benchmarks
302 echo "${BENCHMARK_TARGETS}" | tr ',' '\n' | while IFS=':' read -r relay_name relay_port; do
303 if [ -z "${relay_name}" ] || [ -z "${relay_port}" ]; then
304 echo "WARNING: Skipping invalid target: ${relay_name}:${relay_port}"
305 continue
306 fi
307
308 relay_url="${relay_name}:${relay_port}"
309 output_file="${RUN_DIR}/${relay_name}_results.txt"
310
311 run_benchmark "${relay_name}" "${relay_url}" "${output_file}"
312
313 # Small delay between tests
314 sleep 5
315 done
316
317 # Run graph traversal benchmarks if enabled
318 if [ "${BENCHMARK_GRAPH_TRAVERSAL}" = "true" ]; then
319 echo ""
320 echo "=================================================="
321 echo "Starting Graph Traversal Benchmark Suite"
322 echo "=================================================="
323 echo "This tests 100k pubkeys with 1-1000 follows each"
324 echo "and performs 3-degree traversal queries"
325 echo "=================================================="
326
327 echo "${BENCHMARK_TARGETS}" | tr ',' '\n' | while IFS=':' read -r relay_name relay_port; do
328 if [ -z "${relay_name}" ] || [ -z "${relay_port}" ]; then
329 continue
330 fi
331
332 relay_url="${relay_name}:${relay_port}"
333 output_file="${RUN_DIR}/${relay_name}_graph_traversal_results.txt"
334
335 run_graph_traversal_benchmark "${relay_name}" "${relay_url}" "${output_file}"
336
337 # Longer delay between graph traversal tests (they're more intensive)
338 sleep 10
339 done
340 fi
341
342 # Generate aggregate report
343 generate_aggregate_report
344
345 echo ""
346 echo "=================================================="
347 echo "Benchmark Suite Completed!"
348 echo "=================================================="
349 echo "Results directory: ${RUN_DIR}"
350 echo "Aggregate report: ${RUN_DIR}/aggregate_report.txt"
351 echo ""
352
353 # Display summary
354 if [ -f "${RUN_DIR}/aggregate_report.txt" ]; then
355 echo "Quick Summary:"
356 echo "=============="
357 grep -A 10 "BENCHMARK COMPARISON TABLE" "${RUN_DIR}/aggregate_report.txt" | tail -n +4
358 fi
359
360 echo ""
361 echo "All benchmark files:"
362 ls -la "${RUN_DIR}/"
363 echo ""
364 echo "Benchmark suite finished at: $(date)"