comprehensive-test.sh raw
1 #!/bin/bash
2 #
3 # Comprehensive Negentropy Sync Test Suite
4 # Tests NIP-77 negentropy sync between:
5 # - strfry (client) <-> ORLY (server)
6 # - ORLY (client) <-> ORLY (server) via orly sync CLI bridge
7 #
8 # Strfry has a built-in `sync` command that uses the negentropy protocol.
9 # ORLY serves NIP-77 via its embedded negentropy handler.
10 # The orly CLI can sync between its local Badger DB and a remote relay.
11 #
12 # This script runs from the HOST and uses `docker compose exec` to
13 # interact with containers.
14 #
15 # Scenarios tested:
16 # 1. Seed strfry with events
17 # 2. Strfry pushes events to ORLY (strfry --dir up)
18 # 3. Seed ORLY with new events
19 # 4. Strfry pulls events from ORLY (strfry --dir down)
20 # 5. Bidirectional sync (strfry <-> ORLY)
21 # 6. strfry <-> ORLY consistency verification
22 # 7. Seed orly-relay-2 with events
23 # 8. orly CLI bridge: relay-1 -> relay-2
24 # 9. orly CLI bridge: relay-2 -> relay-1
25 # 10. Three-way consistency verification
26 #
27 # Usage:
28 # cd tests/negentropy
29 # docker compose build
30 # docker compose up -d
31 # ./comprehensive-test.sh
32 # docker compose down -v
33
34 set -euo pipefail
35
36 # Change to the directory containing docker-compose.yml
37 cd "$(dirname "$0")"
38
39 # Configuration
40 STRFRY_WS="ws://strfry:7777"
41 ORLY_WS="ws://orly-relay-1:3334"
42 ORLY2_WS="ws://orly-relay-2:3335"
43 BRIDGE_DB="/tmp/orly-bridge-db"
44 SEED_COUNT=200
45 EXTRA_COUNT=100
46 VERBOSE="${VERBOSE:-false}"
47
48 # Test results
49 PASSED=0
50 FAILED=0
51
52 # Colors
53 RED='\033[0;31m'
54 GREEN='\033[0;32m'
55 YELLOW='\033[1;33m'
56 BLUE='\033[0;34m'
57 NC='\033[0m'
58
59 log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
60 log_pass() { echo -e "${GREEN}[PASS]${NC} $1"; PASSED=$((PASSED + 1)); }
61 log_fail() { echo -e "${RED}[FAIL]${NC} $1"; FAILED=$((FAILED + 1)); }
62 log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
63 log_phase() { echo ""; echo "========================================"; echo -e "${YELLOW}PHASE: $1${NC}"; echo "========================================"; }
64
65 # Run a command in the test-runner container
66 run_test() {
67 docker compose exec -T test-runner sh -c "$1"
68 }
69
70 # Run a command in the strfry container
71 run_strfry() {
72 docker compose exec -T strfry sh -c "$1"
73 }
74
75 # Count events on a relay via WebSocket from test-runner.
76 # Sends a REQ, reads until EOSE, counts EVENT messages.
77 # Usage: count_events <ws_url> [filter_json]
78 count_events() {
79 local url=$1
80 local filter=${2:-'{}'}
81
82 # IMPORTANT: We use { printf ...; sleep 60; } to keep stdin open.
83 # Without this, websocat sends a close frame when stdin EOF is hit,
84 # and the relay may not have sent all events yet.
85 #
86 # awk counts EVENT messages and exits on EOSE (breaking the pipe).
87 # timeout is a safety net in case EOSE never arrives.
88 local result
89 result=$(run_test "{ printf '[\"REQ\",\"c\",%s]\n' '${filter}'; sleep 60; } | timeout 20 websocat '${url}' 2>/dev/null | awk 'BEGIN{c=0;f=0} /EOSE/{f=1; print c; exit} /EVENT/{c++} END{if(f==0) print c}'") || true
90
91 # Trim whitespace; default to 0 if empty
92 result=$(echo "${result}" | tr -d '[:space:]')
93 echo "${result:-0}"
94 }
95
96 # Generate and send events to a relay
97 # Usage: generate_events <relay_ws_url> <count>
98 generate_events() {
99 local url=$1
100 local count=$2
101
102 log_info "Generating $count events and sending to $url ..."
103 run_test "event-generator -count $count -relay '$url' -batch 50" 2>&1 | while IFS= read -r line; do
104 if [ "$VERBOSE" = "true" ]; then
105 echo " $line"
106 fi
107 done
108
109 # Give the relay time to process
110 sleep 3
111 }
112
113 # Wait for a relay to be healthy (via docker compose health check)
114 wait_for_services() {
115 log_info "Checking service health..."
116
117 local services=("strfry" "orly-relay-1" "orly-relay-2" "test-runner")
118 for svc in "${services[@]}"; do
119 local status
120 status=$(docker compose ps --format '{{.Health}}' "$svc" 2>/dev/null || echo "unknown")
121 if [ "$status" = "healthy" ] || [ "$svc" = "test-runner" ]; then
122 log_info " $svc: ready"
123 else
124 log_warn " $svc: $status (may not be ready)"
125 fi
126 done
127 }
128
129 # ============================================================
130 # Phase 1: Seed strfry with events
131 # ============================================================
132 phase1_seed_strfry() {
133 log_phase "1. SEED STRFRY - Generate $SEED_COUNT events"
134
135 generate_events "$STRFRY_WS" "$SEED_COUNT"
136
137 local count
138 count=$(count_events "$STRFRY_WS" '{"limit":10000}')
139 log_info "Strfry has $count events"
140
141 # Replaceable events (kind 0, 3, 10000, 10001) get deduplicated per pubkey,
142 # so stored count is lower than sent count. With 3 test users and ~30%
143 # replaceable kinds, expect roughly 70% stored.
144 local min_expected=$((SEED_COUNT / 2))
145 if [ "$count" -ge "$min_expected" ]; then
146 log_pass "Strfry seeded with $count events (sent $SEED_COUNT, some replaceable)"
147 else
148 log_fail "Strfry only has $count events (expected >= $min_expected from $SEED_COUNT sent)"
149 fi
150 }
151
152 # ============================================================
153 # Phase 2: Strfry pushes events to ORLY
154 # ============================================================
155 phase2_strfry_push_to_orly() {
156 log_phase "2. STRFRY PUSH - Push events from strfry to ORLY"
157
158 local orly_before
159 orly_before=$(count_events "$ORLY_WS" '{"limit":10000}')
160 log_info "ORLY has $orly_before events before sync"
161
162 log_info "Running: strfry sync $ORLY_WS --dir up"
163 run_strfry "/app/strfry --config=/etc/strfry.conf sync $ORLY_WS --dir up" 2>&1 || true
164
165 sleep 5
166
167 local orly_after
168 orly_after=$(count_events "$ORLY_WS" '{"limit":10000}')
169 log_info "ORLY has $orly_after events after sync (was $orly_before)"
170
171 if [ "$orly_after" -gt "$orly_before" ]; then
172 local synced=$((orly_after - orly_before))
173 log_pass "Pushed $synced events from strfry to ORLY"
174 else
175 log_fail "No events pushed to ORLY (still $orly_after)"
176 fi
177 }
178
179 # ============================================================
180 # Phase 3: Seed ORLY with new events
181 # ============================================================
182 phase3_seed_orly() {
183 log_phase "3. SEED ORLY - Generate $EXTRA_COUNT new events on ORLY"
184
185 local orly_before
186 orly_before=$(count_events "$ORLY_WS" '{"limit":10000}')
187 log_info "ORLY has $orly_before events before seeding"
188
189 generate_events "$ORLY_WS" "$EXTRA_COUNT"
190
191 local orly_after
192 orly_after=$(count_events "$ORLY_WS" '{"limit":10000}')
193 log_info "ORLY now has $orly_after events (was $orly_before)"
194
195 if [ "$orly_after" -gt "$orly_before" ]; then
196 local added=$((orly_after - orly_before))
197 log_pass "ORLY stored $added new events ($orly_after total)"
198 else
199 log_fail "ORLY count didn't increase (still $orly_after)"
200 fi
201 }
202
203 # ============================================================
204 # Phase 4: Strfry pulls new events from ORLY
205 # ============================================================
206 phase4_strfry_pull_from_orly() {
207 log_phase "4. STRFRY PULL - Pull new events from ORLY to strfry"
208
209 local strfry_before
210 strfry_before=$(count_events "$STRFRY_WS" '{"limit":10000}')
211 log_info "Strfry has $strfry_before events before sync"
212
213 log_info "Running: strfry sync $ORLY_WS --dir down"
214 run_strfry "/app/strfry --config=/etc/strfry.conf sync $ORLY_WS --dir down" 2>&1 || true
215
216 sleep 5
217
218 local strfry_after
219 strfry_after=$(count_events "$STRFRY_WS" '{"limit":10000}')
220 log_info "Strfry has $strfry_after events after sync (was $strfry_before)"
221
222 if [ "$strfry_after" -gt "$strfry_before" ]; then
223 local synced=$((strfry_after - strfry_before))
224 log_pass "Pulled $synced events from ORLY to strfry"
225 else
226 log_fail "No new events pulled to strfry (still $strfry_after)"
227 fi
228 }
229
230 # ============================================================
231 # Phase 5: Bidirectional sync
232 # ============================================================
233 phase5_bidirectional() {
234 log_phase "5. BIDIRECTIONAL - Sync both directions"
235
236 # Add unique events to both sides
237 log_info "Adding 50 events to strfry..."
238 generate_events "$STRFRY_WS" 50
239
240 log_info "Adding 50 events to ORLY..."
241 generate_events "$ORLY_WS" 50
242
243 local strfry_before orly_before
244 strfry_before=$(count_events "$STRFRY_WS" '{"limit":10000}')
245 orly_before=$(count_events "$ORLY_WS" '{"limit":10000}')
246
247 log_info "Before bidirectional sync: strfry=$strfry_before, ORLY=$orly_before"
248
249 log_info "Running: strfry sync $ORLY_WS --dir both"
250 run_strfry "/app/strfry --config=/etc/strfry.conf sync $ORLY_WS --dir both" 2>&1 || true
251
252 sleep 5
253
254 local strfry_after orly_after
255 strfry_after=$(count_events "$STRFRY_WS" '{"limit":10000}')
256 orly_after=$(count_events "$ORLY_WS" '{"limit":10000}')
257
258 log_info "After bidirectional sync: strfry=$strfry_after, ORLY=$orly_after"
259
260 local diff=$((strfry_after - orly_after))
261 if [ "${diff#-}" -le 50 ]; then
262 log_pass "Bidirectional sync achieved consistency (diff: $diff)"
263 else
264 log_fail "Event counts still differ significantly (diff: $diff)"
265 fi
266 }
267
268 # ============================================================
269 # Phase 6: strfry <-> ORLY consistency verification
270 # ============================================================
271 phase6_strfry_orly_verification() {
272 log_phase "6. STRFRY <-> ORLY VERIFICATION"
273
274 local strfry_total orly_total
275 strfry_total=$(count_events "$STRFRY_WS" '{"limit":10000}')
276 orly_total=$(count_events "$ORLY_WS" '{"limit":10000}')
277
278 log_info "Event counts:"
279 log_info " strfry: $strfry_total"
280 log_info " orly-relay-1: $orly_total"
281
282 # Both should have a reasonable number of events
283 if [ "$strfry_total" -gt 0 ] && [ "$orly_total" -gt 0 ]; then
284 log_pass "Both relays have events (strfry=$strfry_total, ORLY=$orly_total)"
285 else
286 log_fail "One or both relays are empty"
287 fi
288
289 # Check consistency
290 local diff=$((strfry_total - orly_total))
291 if [ "${diff#-}" -le 50 ]; then
292 log_pass "strfry and ORLY are consistent (diff: $diff)"
293 else
294 log_warn "strfry and ORLY differ by $diff events"
295 fi
296 }
297
298 # ============================================================
299 # Phase 7: Seed orly-relay-2 with events
300 # ============================================================
301 phase7_seed_orly2() {
302 log_phase "7. SEED ORLY-RELAY-2 - Generate $SEED_COUNT events"
303
304 generate_events "$ORLY2_WS" "$SEED_COUNT"
305
306 local count
307 count=$(count_events "$ORLY2_WS" '{"limit":10000}')
308 log_info "orly-relay-2 has $count events"
309
310 local min_expected=$((SEED_COUNT / 2))
311 if [ "$count" -ge "$min_expected" ]; then
312 log_pass "orly-relay-2 seeded with $count events (sent $SEED_COUNT, some replaceable)"
313 else
314 log_fail "orly-relay-2 only has $count events (expected >= $min_expected)"
315 fi
316 }
317
318 # ============================================================
319 # Phase 8: orly CLI bridge: relay-1 -> relay-2
320 # Uses orly sync CLI with a temporary Badger DB to bridge events
321 # from orly-relay-1 to orly-relay-2 (tests orly as NIP-77 client
322 # against orly as NIP-77 server).
323 # ============================================================
324 phase8_orly_bridge_r1_to_r2() {
325 log_phase "8. ORLY CLI BRIDGE - relay-1 -> relay-2"
326
327 local orly2_before
328 orly2_before=$(count_events "$ORLY2_WS" '{"limit":10000}')
329 log_info "orly-relay-2 has $orly2_before events before bridge sync"
330
331 # Clean bridge DB
332 run_test "rm -rf $BRIDGE_DB" || true
333
334 # Step 1: Pull events from relay-1 into bridge DB
335 log_info "Step 1: orly sync (pull from relay-1 into bridge DB)"
336 run_test "ORLY_LOG_LEVEL=info orly sync $ORLY_WS --data-dir $BRIDGE_DB" 2>&1 || true
337
338 sleep 3
339
340 # Step 2: Push bridge DB events to relay-2
341 log_info "Step 2: orly sync (push bridge DB to relay-2)"
342 run_test "ORLY_LOG_LEVEL=info orly sync $ORLY2_WS --data-dir $BRIDGE_DB" 2>&1 || true
343
344 sleep 5
345
346 local orly2_after
347 orly2_after=$(count_events "$ORLY2_WS" '{"limit":10000}')
348 log_info "orly-relay-2 has $orly2_after events after bridge sync (was $orly2_before)"
349
350 if [ "$orly2_after" -gt "$orly2_before" ]; then
351 local synced=$((orly2_after - orly2_before))
352 log_pass "Bridged $synced events from relay-1 to relay-2 via orly CLI"
353 else
354 log_fail "No events bridged to relay-2 (still $orly2_after)"
355 fi
356 }
357
358 # ============================================================
359 # Phase 9: orly CLI bridge: relay-2 -> relay-1
360 # The bridge DB already has relay-2 events from Phase 8 (bidirectional
361 # sync picked them up). Sync with relay-1 to push relay-2's events.
362 # ============================================================
363 phase9_orly_bridge_r2_to_r1() {
364 log_phase "9. ORLY CLI BRIDGE - relay-2 -> relay-1"
365
366 local orly1_before
367 orly1_before=$(count_events "$ORLY_WS" '{"limit":10000}')
368 log_info "orly-relay-1 has $orly1_before events before bridge sync"
369
370 # Bridge DB already has relay-2 events from Phase 8 step 2 (bidirectional).
371 # Sync with relay-1 to push those events.
372 log_info "Syncing bridge DB with relay-1 (pushes relay-2 events)"
373 run_test "ORLY_LOG_LEVEL=info orly sync $ORLY_WS --data-dir $BRIDGE_DB" 2>&1 || true
374
375 sleep 5
376
377 local orly1_after
378 orly1_after=$(count_events "$ORLY_WS" '{"limit":10000}')
379 log_info "orly-relay-1 has $orly1_after events after bridge sync (was $orly1_before)"
380
381 if [ "$orly1_after" -gt "$orly1_before" ]; then
382 local synced=$((orly1_after - orly1_before))
383 log_pass "Bridged $synced events from relay-2 to relay-1 via orly CLI"
384 else
385 log_fail "No events bridged to relay-1 (still $orly1_after)"
386 fi
387
388 # Clean up bridge DB
389 run_test "rm -rf $BRIDGE_DB" || true
390 }
391
392 # ============================================================
393 # Phase 10: Three-way consistency verification
394 # ============================================================
395 phase10_three_way_verification() {
396 log_phase "10. THREE-WAY CONSISTENCY VERIFICATION"
397
398 local strfry_total orly1_total orly2_total
399 strfry_total=$(count_events "$STRFRY_WS" '{"limit":10000}')
400 orly1_total=$(count_events "$ORLY_WS" '{"limit":10000}')
401 orly2_total=$(count_events "$ORLY2_WS" '{"limit":10000}')
402
403 log_info "Final event counts:"
404 log_info " strfry: $strfry_total"
405 log_info " orly-relay-1: $orly1_total"
406 log_info " orly-relay-2: $orly2_total"
407
408 # All three should have events
409 if [ "$strfry_total" -gt 0 ] && [ "$orly1_total" -gt 0 ] && [ "$orly2_total" -gt 0 ]; then
410 log_pass "All three relays have events"
411 else
412 log_fail "One or more relays are empty"
413 fi
414
415 # Check orly-relay-1 vs orly-relay-2 consistency
416 local diff_orly=$((orly1_total - orly2_total))
417 if [ "${diff_orly#-}" -le 50 ]; then
418 log_pass "orly-relay-1 and orly-relay-2 are consistent (diff: $diff_orly)"
419 else
420 log_fail "orly relays differ significantly (diff: $diff_orly)"
421 fi
422
423 # Check strfry vs orly-relay-1 consistency
424 local diff_strfry=$((strfry_total - orly1_total))
425 if [ "${diff_strfry#-}" -le 50 ]; then
426 log_pass "strfry and orly-relay-1 are consistent (diff: $diff_strfry)"
427 else
428 log_warn "strfry and orly-relay-1 differ by $diff_strfry events"
429 fi
430 }
431
432 # ============================================================
433 # Main
434 # ============================================================
435 main() {
436 echo "========================================"
437 echo "Negentropy (NIP-77) Interop Test Suite"
438 echo "strfry <-> ORLY <-> ORLY"
439 echo "========================================"
440 echo ""
441 echo "Config:"
442 echo " Seed events: $SEED_COUNT"
443 echo " Extra events: $EXTRA_COUNT"
444 echo ""
445
446 wait_for_services
447
448 # Part 1: strfry <-> ORLY interop (phases 1-6)
449 phase1_seed_strfry
450 phase2_strfry_push_to_orly
451 phase3_seed_orly
452 phase4_strfry_pull_from_orly
453 phase5_bidirectional
454 phase6_strfry_orly_verification
455
456 # Part 2: ORLY <-> ORLY interop via CLI bridge (phases 7-10)
457 phase7_seed_orly2
458 phase8_orly_bridge_r1_to_r2
459 phase9_orly_bridge_r2_to_r1
460 phase10_three_way_verification
461
462 echo ""
463 echo "========================================"
464 echo "TEST SUMMARY"
465 echo "========================================"
466 echo -e "${GREEN}Passed: $PASSED${NC}"
467 echo -e "${RED}Failed: $FAILED${NC}"
468 echo ""
469
470 if [ "$FAILED" -eq 0 ]; then
471 echo -e "${GREEN}All tests passed!${NC}"
472 exit 0
473 else
474 echo -e "${RED}Some tests failed.${NC}"
475 exit 1
476 fi
477 }
478
479 case "${1:-}" in
480 --verbose|-v)
481 VERBOSE=true
482 main
483 ;;
484 --help|-h)
485 echo "Usage: $0 [--verbose|-v] [--help|-h]"
486 echo ""
487 echo "Run from the tests/negentropy directory with containers up:"
488 echo " docker compose build"
489 echo " docker compose up -d"
490 echo " $0"
491 echo " docker compose down -v"
492 exit 0
493 ;;
494 *)
495 main
496 ;;
497 esac
498