run_benchmarks.sh raw
1 #!/bin/bash
2
3 # Benchmark runner script for secp256k1 implementation comparison
4 # Runs benchmarks multiple times and generates statistical analysis
5
6 set -e
7
8 echo "=========================================="
9 echo "secp256k1 Implementation Benchmark Suite"
10 echo "=========================================="
11 echo ""
12
13 # Check for dependencies
14 echo "Checking dependencies..."
15
16 if ! command -v go &> /dev/null; then
17 echo "Error: Go is not installed"
18 exit 1
19 fi
20
21 if ! command -v benchstat &> /dev/null; then
22 echo "Installing benchstat for statistical analysis..."
23 go install golang.org/x/perf/cmd/benchstat@latest
24 fi
25
26 # Check if libsecp256k1 is available
27 if ! ldconfig -p | grep -q libsecp256k1; then
28 echo "Warning: libsecp256k1 not found. P8K benchmarks may be skipped."
29 echo "Install with: sudo apt-get install libsecp256k1-dev (Ubuntu/Debian)"
30 echo "or: brew install libsecp256k1 (macOS)"
31 echo ""
32 fi
33
34 # Configuration
35 BENCHTIME=${BENCHTIME:-3s}
36 COUNT=${COUNT:-1}
37 OUTPUT_DIR="results"
38 TIMESTAMP=$(date +%Y%m%d_%H%M%S)
39
40 echo "Benchmark configuration:"
41 echo " Duration: $BENCHTIME per benchmark"
42 echo " Iterations: $COUNT runs"
43 echo " Output directory: $OUTPUT_DIR"
44 echo ""
45
46 # Create output directory
47 mkdir -p "$OUTPUT_DIR"
48
49 # Function to run benchmarks
50 run_benchmark() {
51 local name=$1
52 local bench_pattern=$2
53 local output_file="$OUTPUT_DIR/${name}_${TIMESTAMP}.txt"
54
55 echo "Running: $name"
56 echo " Output: $output_file"
57
58 go test -bench="$bench_pattern" \
59 -benchmem \
60 -benchtime="$BENCHTIME" \
61 -count="$COUNT" \
62 2>&1 | tee "$output_file"
63
64 echo "✓ Completed: $name"
65 echo ""
66 }
67
68 # Run all benchmarks
69 echo "=========================================="
70 echo "Running Benchmarks"
71 echo "=========================================="
72 echo ""
73
74 run_benchmark "all_operations" "BenchmarkAll"
75 run_benchmark "pubkey_derivation" "BenchmarkComparative_PubkeyDerivation"
76 run_benchmark "schnorr_sign" "BenchmarkComparative_SchnorrSign"
77 run_benchmark "schnorr_verify" "BenchmarkComparative_SchnorrVerify"
78 run_benchmark "ecdh" "BenchmarkComparative_ECDH"
79
80 # Run individual implementation benchmarks
81 run_benchmark "btcec_only" "BenchmarkBTCEC"
82 run_benchmark "p256k1_only" "BenchmarkP256K1"
83 run_benchmark "p8k_only" "BenchmarkP8K"
84
85 # Generate statistical analysis
86 echo "=========================================="
87 echo "Generating Statistical Analysis"
88 echo "=========================================="
89 echo ""
90
91 for file in "$OUTPUT_DIR"/*_${TIMESTAMP}.txt; do
92 if [ -f "$file" ]; then
93 basename=$(basename "$file" .txt)
94 echo "Analysis: $basename"
95 benchstat "$file" | tee "$OUTPUT_DIR/${basename}_stats.txt"
96 echo ""
97 fi
98 done
99
100 # Generate comparison report
101 COMPARISON_FILE="$OUTPUT_DIR/comparison_${TIMESTAMP}.txt"
102 echo "=========================================="
103 echo "Implementation Comparison Summary"
104 echo "=========================================="
105 echo ""
106
107 echo "Comparison between implementations" > "$COMPARISON_FILE"
108 echo "Generated: $(date)" >> "$COMPARISON_FILE"
109 echo "" >> "$COMPARISON_FILE"
110
111 # Compare each operation
112 for op in pubkey_derivation schnorr_sign schnorr_verify ecdh; do
113 file="$OUTPUT_DIR/${op}_${TIMESTAMP}.txt"
114 if [ -f "$file" ]; then
115 echo "=== $op ===" >> "$COMPARISON_FILE"
116 benchstat "$file" >> "$COMPARISON_FILE"
117 echo "" >> "$COMPARISON_FILE"
118 fi
119 done
120
121 cat "$COMPARISON_FILE"
122
123 echo "=========================================="
124 echo "Benchmark Results Summary"
125 echo "=========================================="
126 echo ""
127 echo "Results saved to: $OUTPUT_DIR"
128 echo ""
129 echo "Files generated:"
130 ls -lh "$OUTPUT_DIR"/*_${TIMESTAMP}* | awk '{print " " $9 " (" $5 ")"}'
131 echo ""
132
133 # Generate markdown report
134 MARKDOWN_FILE="$OUTPUT_DIR/REPORT_${TIMESTAMP}.md"
135 echo "Generating markdown report: $MARKDOWN_FILE"
136
137 cat > "$MARKDOWN_FILE" << 'EOF'
138 # secp256k1 Implementation Benchmark Results
139
140 ## Test Environment
141
142 EOF
143
144 echo "- **Date**: $(date)" >> "$MARKDOWN_FILE"
145 echo "- **Go Version**: $(go version)" >> "$MARKDOWN_FILE"
146 echo "- **OS**: $(uname -s) $(uname -r)" >> "$MARKDOWN_FILE"
147 echo "- **CPU**: $(grep -m1 "model name" /proc/cpuinfo 2>/dev/null | cut -d: -f2 | xargs || echo "Unknown")" >> "$MARKDOWN_FILE"
148 echo "- **Benchmark Time**: $BENCHTIME per test" >> "$MARKDOWN_FILE"
149 echo "- **Iterations**: $COUNT runs" >> "$MARKDOWN_FILE"
150 echo "" >> "$MARKDOWN_FILE"
151
152 cat >> "$MARKDOWN_FILE" << 'EOF'
153 ## Implementations Tested
154
155 1. **BTCEC** - btcsuite/btcd implementation (pure Go)
156 2. **P256K1** - mleku/p256k1 implementation (pure Go)
157 3. **P8K** - p8k.mleku.dev implementation (purego, C bindings)
158
159 ## Results
160
161 EOF
162
163 # Add results from comparison file
164 cat "$COMPARISON_FILE" >> "$MARKDOWN_FILE"
165
166 echo "" >> "$MARKDOWN_FILE"
167 echo "## Raw Data" >> "$MARKDOWN_FILE"
168 echo "" >> "$MARKDOWN_FILE"
169 echo "Full benchmark results are available in:" >> "$MARKDOWN_FILE"
170 echo "" >> "$MARKDOWN_FILE"
171 for file in "$OUTPUT_DIR"/*_${TIMESTAMP}.txt; do
172 if [ -f "$file" ]; then
173 echo "- $(basename "$file")" >> "$MARKDOWN_FILE"
174 fi
175 done
176
177 echo ""
178 echo "✓ Markdown report generated: $MARKDOWN_FILE"
179 echo ""
180 echo "=========================================="
181 echo "Benchmark suite completed!"
182 echo "=========================================="
183
184