benchmark.sh raw
1 #!/bin/bash
2 set -euo pipefail
3
4 # scripts/benchmark.sh - Run full benchmark suite on a relay at a configurable address
5 #
6 # Usage:
7 # ./scripts/benchmark.sh [relay_address] [relay_port]
8 #
9 # Example:
10 # ./scripts/benchmark.sh localhost 3334
11 # ./scripts/benchmark.sh nostr.example.com 8080
12 #
13 # If relay_address and relay_port are not provided, defaults to localhost:3334
14
15 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
16 REPO_ROOT="$(cd -- "${SCRIPT_DIR}/.." && pwd)"
17 cd "$REPO_ROOT"
18
19 # Default values
20 RELAY_ADDRESS="${1:-localhost}"
21 RELAY_PORT="${2:-3334}"
22 RELAY_URL="ws://${RELAY_ADDRESS}:${RELAY_PORT}"
23 BENCHMARK_EVENTS="${BENCHMARK_EVENTS:-10000}"
24 BENCHMARK_WORKERS="${BENCHMARK_WORKERS:-8}"
25 BENCHMARK_DURATION="${BENCHMARK_DURATION:-60s}"
26 REPORTS_DIR="${REPORTS_DIR:-$REPO_ROOT/cmd/benchmark/reports}"
27 TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
28 RUN_DIR="${REPORTS_DIR}/run_${TIMESTAMP}"
29
30 # Ensure the benchmark binary is built
31 BENCHMARK_BIN="${REPO_ROOT}/cmd/benchmark/benchmark"
32 if [[ ! -x "$BENCHMARK_BIN" ]]; then
33 echo "Building benchmark binary (pure Go + purego)..."
34 cd "$REPO_ROOT/cmd/benchmark"
35 CGO_ENABLED=0 go build -o "$BENCHMARK_BIN" .
36 # Copy libsecp256k1.so from repo root (runtime optional)
37 if [[ -f "$REPO_ROOT/libsecp256k1.so" ]]; then
38 cp "$REPO_ROOT/libsecp256k1.so" "$(dirname "$BENCHMARK_BIN")/"
39 chmod +x "$(dirname "$BENCHMARK_BIN")/libsecp256k1.so" 2>/dev/null || true
40 fi
41 cd "$REPO_ROOT"
42 fi
43
44 # Create output directory
45 mkdir -p "${RUN_DIR}"
46
47 echo "=================================================="
48 echo "Nostr Relay Benchmark"
49 echo "=================================================="
50 echo "Timestamp: $(date)"
51 echo "Target Relay: ${RELAY_URL}"
52 echo "Events per test: ${BENCHMARK_EVENTS}"
53 echo "Concurrent workers: ${BENCHMARK_WORKERS}"
54 echo "Test duration: ${BENCHMARK_DURATION}"
55 echo "Output directory: ${RUN_DIR}"
56 echo "=================================================="
57
58 # Function to wait for relay to be ready
59 wait_for_relay() {
60 local url="$1"
61 local max_attempts=30
62 local attempt=0
63
64 echo "Waiting for relay to be ready at ${url}..."
65
66 while [ $attempt -lt $max_attempts ]; do
67 # Try to get HTTP status code with curl
68 local status=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout 5 --max-time 5 "http://${RELAY_ADDRESS}:${RELAY_PORT}" || echo 000)
69
70 case "$status" in
71 101|200|400|404|426)
72 echo "Relay is ready! (HTTP ${status})"
73 return 0
74 ;;
75 esac
76
77 attempt=$((attempt + 1))
78 echo " Attempt ${attempt}/${max_attempts}: Relay not ready yet (HTTP ${status})..."
79 sleep 2
80 done
81
82 echo "ERROR: Relay failed to become ready after ${max_attempts} attempts"
83 return 1
84 }
85
86 # Function to run benchmark against the relay
87 run_benchmark() {
88 local output_file="${RUN_DIR}/benchmark_results.txt"
89
90 echo ""
91 echo "=================================================="
92 echo "Testing relay at ${RELAY_URL}"
93 echo "=================================================="
94
95 # Wait for relay to be ready
96 if ! wait_for_relay "${RELAY_ADDRESS}:${RELAY_PORT}"; then
97 echo "ERROR: Relay is not responding, aborting..."
98 echo "RELAY_URL: ${RELAY_URL}" > "${output_file}"
99 echo "STATUS: FAILED - Relay not responding" >> "${output_file}"
100 echo "ERROR: Connection failed" >> "${output_file}"
101 return 1
102 fi
103
104 # Run the benchmark
105 echo "Running benchmark against ${RELAY_URL}..."
106
107 # Create temporary directory for benchmark data
108 TEMP_DATA_DIR="/tmp/benchmark_${TIMESTAMP}"
109 mkdir -p "${TEMP_DATA_DIR}"
110
111 # Run benchmark and capture both stdout and stderr
112 if "${BENCHMARK_BIN}" \
113 -relay-url="${RELAY_URL}" \
114 -datadir="${TEMP_DATA_DIR}" \
115 -events="${BENCHMARK_EVENTS}" \
116 -workers="${BENCHMARK_WORKERS}" \
117 -duration="${BENCHMARK_DURATION}" \
118 # > "${output_file}"
119 2>&1; then
120 echo "✓ Benchmark completed successfully"
121 # Add relay identification to the report
122 echo "" >> "${output_file}"
123 echo "RELAY_URL: ${RELAY_URL}" >> "${output_file}"
124 echo "TEST_TIMESTAMP: $(date -Iseconds)" >> "${output_file}"
125 echo "BENCHMARK_CONFIG:" >> "${output_file}"
126 echo " Events: ${BENCHMARK_EVENTS}" >> "${output_file}"
127 echo " Workers: ${BENCHMARK_WORKERS}" >> "${output_file}"
128 echo " Duration: ${BENCHMARK_DURATION}" >> "${output_file}" else
129 echo "✗ Benchmark failed"
130 echo "" >> "${output_file}"
131 echo "RELAY_URL: ${RELAY_URL}" >> "${output_file}"
132 echo "STATUS: FAILED" >> "${output_file}"
133 echo "TEST_TIMESTAMP: $(date -Iseconds)" >> "${output_file}"
134 fi
135
136 # Clean up temporary data
137 rm -rf "${TEMP_DATA_DIR}"
138
139 return 0
140 }
141
142 # Main execution
143 echo "Starting relay benchmark..."
144 run_benchmark
145
146 # Display results
147 if [ -f "${RUN_DIR}/benchmark_results.txt" ]; then
148 echo ""
149 echo "=================================================="
150 echo "Benchmark Results Summary"
151 echo "=================================================="
152 # Extract key metrics from the benchmark report
153 if grep -q "STATUS: FAILED" "${RUN_DIR}/benchmark_results.txt"; then
154 echo "Status: FAILED"
155 grep "ERROR:" "${RUN_DIR}/benchmark_results.txt" | head -1 || echo "Error: Unknown failure"
156 else
157 echo "Status: COMPLETED"
158
159 # Extract performance metrics
160 grep "Events/sec:" "${RUN_DIR}/benchmark_results.txt" | head -3 || true
161 grep "Success Rate:" "${RUN_DIR}/benchmark_results.txt" | head -3 || true
162 grep "Avg Latency:" "${RUN_DIR}/benchmark_results.txt" | head -3 || true
163 grep "P95 Latency:" "${RUN_DIR}/benchmark_results.txt" | head -3 || true
164 grep "Memory:" "${RUN_DIR}/benchmark_results.txt" | head -3 || true
165 fi
166
167 echo ""
168 echo "Full results available in: ${RUN_DIR}/benchmark_results.txt"
169 fi
170
171 echo ""
172 echo "=================================================="
173 echo "Benchmark Completed!"
174 echo "=================================================="
175 echo "Results directory: ${RUN_DIR}"
176 echo "Benchmark finished at: $(date)"