migrate-badger-config.sh raw
1 #!/bin/bash
2 # Badger Database Migration Script
3 # Migrates ORLY database to new Badger configuration with VLogPercentile optimization
4
5 set -e # Exit on error
6
7 # Colors for output
8 RED='\033[0;31m'
9 GREEN='\033[0;32m'
10 YELLOW='\033[1;33m'
11 NC='\033[0m' # No Color
12
13 echo -e "${GREEN}=== ORLY Badger Database Migration ===${NC}"
14 echo ""
15
16 # Configuration
17 DATA_DIR="${ORLY_DATA_DIR:-$HOME/.local/share/ORLY}"
18 BACKUP_DIR="${DATA_DIR}-backup-$(date +%Y%m%d-%H%M%S)"
19 EXPORT_FILE="${DATA_DIR}/events-export.jsonl"
20 RELAY_BIN="${RELAY_BIN:-./orly}"
21
22 # Check if relay binary exists
23 if [ ! -f "$RELAY_BIN" ]; then
24 echo -e "${RED}Error: ORLY binary not found at $RELAY_BIN${NC}"
25 echo "Please build the relay first: go build -o orly"
26 echo "Or set RELAY_BIN environment variable to the binary location"
27 exit 1
28 fi
29
30 # Check if database exists
31 if [ ! -d "$DATA_DIR" ]; then
32 echo -e "${YELLOW}Warning: Database directory not found at $DATA_DIR${NC}"
33 echo "Nothing to migrate. If this is a fresh install, you can skip migration."
34 exit 0
35 fi
36
37 # Check disk space
38 DB_SIZE=$(du -sb "$DATA_DIR" | cut -f1)
39 AVAILABLE_SPACE=$(df "$HOME" | tail -1 | awk '{print $4}')
40 AVAILABLE_SPACE=$((AVAILABLE_SPACE * 1024)) # Convert to bytes
41 REQUIRED_SPACE=$((DB_SIZE * 3)) # 3x for safety (export + backup + new DB)
42
43 echo "Database size: $(numfmt --to=iec-i --suffix=B $DB_SIZE)"
44 echo "Available space: $(numfmt --to=iec-i --suffix=B $AVAILABLE_SPACE)"
45 echo "Required space: $(numfmt --to=iec-i --suffix=B $REQUIRED_SPACE)"
46 echo ""
47
48 if [ $AVAILABLE_SPACE -lt $REQUIRED_SPACE ]; then
49 echo -e "${RED}Error: Not enough disk space!${NC}"
50 echo "Required: $(numfmt --to=iec-i --suffix=B $REQUIRED_SPACE)"
51 echo "Available: $(numfmt --to=iec-i --suffix=B $AVAILABLE_SPACE)"
52 echo ""
53 echo "Options:"
54 echo " 1. Free up disk space"
55 echo " 2. Use natural compaction (no migration needed)"
56 echo " 3. Export to external drive and import back"
57 exit 1
58 fi
59
60 # Check if relay is running
61 if pgrep -x "orly" > /dev/null; then
62 echo -e "${YELLOW}Warning: ORLY relay is currently running${NC}"
63 echo "The relay should be stopped before migration."
64 echo ""
65 read -p "Stop the relay now? (y/N) " -n 1 -r
66 echo
67 if [[ $REPLY =~ ^[Yy]$ ]]; then
68 echo "Attempting to stop relay..."
69 if systemctl is-active --quiet orly; then
70 sudo systemctl stop orly
71 echo -e "${GREEN}Relay stopped via systemd${NC}"
72 else
73 pkill orly
74 sleep 2
75 if pgrep -x "orly" > /dev/null; then
76 echo -e "${RED}Failed to stop relay. Please stop it manually and try again.${NC}"
77 exit 1
78 fi
79 echo -e "${GREEN}Relay stopped${NC}"
80 fi
81 else
82 echo "Please stop the relay and run this script again."
83 exit 1
84 fi
85 fi
86
87 echo ""
88 echo -e "${YELLOW}=== Migration Plan ===${NC}"
89 echo "1. Export all events to JSONL: $EXPORT_FILE"
90 echo "2. Backup current database to: $BACKUP_DIR"
91 echo "3. Create new database with optimized configuration"
92 echo "4. Import all events (rebuilds indexes)"
93 echo "5. Verify event counts match"
94 echo ""
95 echo "Estimated time: $(( (DB_SIZE / 1024 / 1024 / 100) + 1 )) - $(( (DB_SIZE / 1024 / 1024 / 50) + 1 )) minutes"
96 echo ""
97 read -p "Proceed with migration? (y/N) " -n 1 -r
98 echo
99 if [[ ! $REPLY =~ ^[Yy]$ ]]; then
100 echo "Migration cancelled."
101 exit 0
102 fi
103
104 # Step 1: Export events
105 echo ""
106 echo -e "${GREEN}=== Step 1: Exporting Events ===${NC}"
107 echo "This may take several minutes for large databases..."
108 echo ""
109
110 # We'll use a Go program to export since the binary doesn't have a CLI export command
111 # Create temporary export program
112 EXPORT_PROG=$(mktemp -d)/export-db.go
113 cat > "$EXPORT_PROG" << 'EOF'
114 package main
115
116 import (
117 "context"
118 "fmt"
119 "os"
120 "next.orly.dev/pkg/database"
121 )
122
123 func main() {
124 if len(os.Args) < 3 {
125 fmt.Fprintf(os.Stderr, "Usage: %s <data-dir> <output-file>\n", os.Args[0])
126 os.Exit(1)
127 }
128
129 dataDir := os.Args[1]
130 outFile := os.Args[2]
131
132 ctx := context.Background()
133 cancel := func() {}
134
135 db, err := database.New(ctx, cancel, dataDir, "error")
136 if err != nil {
137 fmt.Fprintf(os.Stderr, "Failed to open database: %v\n", err)
138 os.Exit(1)
139 }
140 defer db.Close()
141
142 f, err := os.Create(outFile)
143 if err != nil {
144 fmt.Fprintf(os.Stderr, "Failed to create output file: %v\n", err)
145 os.Exit(1)
146 }
147 defer f.Close()
148
149 fmt.Println("Exporting events...")
150 db.Export(ctx, f)
151 fmt.Println("Export complete!")
152 }
153 EOF
154
155 # Build and run export program
156 echo "Building export tool..."
157 EXPORT_BIN=$(mktemp)
158 if ! go build -o "$EXPORT_BIN" "$EXPORT_PROG" 2>&1; then
159 echo -e "${RED}Failed to build export tool${NC}"
160 rm -f "$EXPORT_PROG" "$EXPORT_BIN"
161 exit 1
162 fi
163
164 echo "Running export..."
165 if ! "$EXPORT_BIN" "$DATA_DIR" "$EXPORT_FILE"; then
166 echo -e "${RED}Export failed!${NC}"
167 rm -f "$EXPORT_PROG" "$EXPORT_BIN"
168 exit 1
169 fi
170
171 rm -f "$EXPORT_PROG" "$EXPORT_BIN"
172
173 # Count exported events
174 EXPORT_COUNT=$(wc -l < "$EXPORT_FILE")
175 echo -e "${GREEN}Exported $EXPORT_COUNT events${NC}"
176 echo "Export size: $(du -h "$EXPORT_FILE" | cut -f1)"
177
178 # Step 2: Backup current database
179 echo ""
180 echo -e "${GREEN}=== Step 2: Backing Up Current Database ===${NC}"
181 echo "Moving $DATA_DIR to $BACKUP_DIR"
182 mv "$DATA_DIR" "$BACKUP_DIR"
183 echo -e "${GREEN}Backup complete${NC}"
184
185 # Step 3 & 4: Create new database and import
186 echo ""
187 echo -e "${GREEN}=== Step 3 & 4: Creating New Database and Importing ===${NC}"
188 echo "This will take longer as indexes are rebuilt..."
189 echo ""
190
191 # Create temporary import program
192 IMPORT_PROG=$(mktemp -d)/import-db.go
193 cat > "$IMPORT_PROG" << 'EOF'
194 package main
195
196 import (
197 "context"
198 "fmt"
199 "os"
200 "next.orly.dev/pkg/database"
201 )
202
203 func main() {
204 if len(os.Args) < 3 {
205 fmt.Fprintf(os.Stderr, "Usage: %s <data-dir> <import-file>\n", os.Args[0])
206 os.Exit(1)
207 }
208
209 dataDir := os.Args[1]
210 importFile := os.Args[2]
211
212 ctx := context.Background()
213 cancel := func() {}
214
215 // This will create new database with updated configuration from database.go
216 db, err := database.New(ctx, cancel, dataDir, "info")
217 if err != nil {
218 fmt.Fprintf(os.Stderr, "Failed to create database: %v\n", err)
219 os.Exit(1)
220 }
221 defer db.Close()
222
223 f, err := os.Open(importFile)
224 if err != nil {
225 fmt.Fprintf(os.Stderr, "Failed to open import file: %v\n", err)
226 os.Exit(1)
227 }
228 defer f.Close()
229
230 fmt.Println("Importing events (this may take a while)...")
231 db.Import(f)
232
233 // Wait for import to complete
234 fmt.Println("Import started. Waiting for completion...")
235 fmt.Println("Check the log output above for progress (logged every 100 events)")
236 }
237 EOF
238
239 # Build and run import program
240 echo "Building import tool..."
241 IMPORT_BIN=$(mktemp)
242 if ! go build -o "$IMPORT_BIN" "$IMPORT_PROG" 2>&1; then
243 echo -e "${RED}Failed to build import tool${NC}"
244 echo "Rolling back..."
245 mv "$BACKUP_DIR" "$DATA_DIR"
246 rm -f "$IMPORT_PROG" "$IMPORT_BIN"
247 exit 1
248 fi
249
250 echo "Running import..."
251 if ! "$IMPORT_BIN" "$DATA_DIR" "$EXPORT_FILE"; then
252 echo -e "${RED}Import failed!${NC}"
253 echo "Rolling back..."
254 rm -rf "$DATA_DIR"
255 mv "$BACKUP_DIR" "$DATA_DIR"
256 rm -f "$IMPORT_PROG" "$IMPORT_BIN"
257 exit 1
258 fi
259
260 rm -f "$IMPORT_PROG" "$IMPORT_BIN"
261
262 # Give import goroutine time to process
263 echo "Waiting for import to complete..."
264 sleep 10
265
266 # Step 5: Verify
267 echo ""
268 echo -e "${GREEN}=== Step 5: Verification ===${NC}"
269
270 NEW_DB_SIZE=$(du -sb "$DATA_DIR" | cut -f1)
271 echo "Old database size: $(numfmt --to=iec-i --suffix=B $DB_SIZE)"
272 echo "New database size: $(numfmt --to=iec-i --suffix=B $NEW_DB_SIZE)"
273 echo ""
274
275 if [ $NEW_DB_SIZE -lt $((DB_SIZE / 10)) ]; then
276 echo -e "${YELLOW}Warning: New database is suspiciously small${NC}"
277 echo "This may indicate an incomplete import."
278 echo "Check the logs in $DATA_DIR/migration.log"
279 echo ""
280 read -p "Continue anyway? (y/N) " -n 1 -r
281 echo
282 if [[ ! $REPLY =~ ^[Yy]$ ]]; then
283 echo "Rolling back..."
284 rm -rf "$DATA_DIR"
285 mv "$BACKUP_DIR" "$DATA_DIR"
286 exit 1
287 fi
288 fi
289
290 echo -e "${GREEN}=== Migration Complete! ===${NC}"
291 echo ""
292 echo "Summary:"
293 echo " - Exported: $EXPORT_COUNT events"
294 echo " - Old DB size: $(numfmt --to=iec-i --suffix=B $DB_SIZE)"
295 echo " - New DB size: $(numfmt --to=iec-i --suffix=B $NEW_DB_SIZE)"
296 echo " - Space saved: $(numfmt --to=iec-i --suffix=B $((DB_SIZE - NEW_DB_SIZE)))"
297 echo " - Backup location: $BACKUP_DIR"
298 echo ""
299 echo "Next steps:"
300 echo " 1. Start the relay: sudo systemctl start orly (or ./orly)"
301 echo " 2. Monitor performance for 24-48 hours"
302 echo " 3. Watch for cache hit ratio >85% in logs"
303 echo " 4. Verify event count and queries work correctly"
304 echo " 5. After verification, remove backup: rm -rf $BACKUP_DIR"
305 echo ""
306 echo "Rollback (if needed):"
307 echo " Stop relay, then: rm -rf $DATA_DIR && mv $BACKUP_DIR $DATA_DIR"
308 echo ""
309