backup.go raw
1 package blossom
2
3 import (
4 "archive/tar"
5 "crypto/sha256"
6 "encoding/hex"
7 "encoding/json"
8 "errors"
9 "fmt"
10 "io"
11 "os"
12 "path/filepath"
13 "sort"
14 "strings"
15 "time"
16
17 "git.smesh.lol/torly/pkg/nostr"
18 )
19
20 // Blossom backup format:
21 //
22 // manifest.json
23 // blobs/<sha256>
24 //
25 // The manifest is the first tar entry. It is deliberately plain JSON so the
26 // orly Badger extractor can emit exactly the same archive.
27 const (
28 backupVersion = 1
29 backupManifest = "manifest.json"
30 backupBlobPrefix = "blobs/"
31 )
32
33 // BackupManifest is the metadata half of a Blossom backup.
34 type BackupManifest struct {
35 Version int `json:"version"`
36 Blobs []BackupEntry `json:"blobs"`
37 }
38
39 // BackupEntry describes one blob in the backup manifest.
40 type BackupEntry struct {
41 SHA256 string `json:"sha256"`
42 PubKey string `json:"pubkey,omitempty"`
43 Size int64 `json:"size"`
44 Type string `json:"type"`
45 Created int64 `json:"created"`
46 }
47
48 // BackupStats is returned by Restore.
49 type BackupStats struct {
50 Restored int `json:"restored"`
51 Skipped int `json:"skipped"`
52 }
53
54 // Snapshot returns a stable copy of the blob metadata.
55 func (s *Server) Snapshot() []Blob {
56 s.mu.Lock()
57 defer s.mu.Unlock()
58 out := make([]Blob, 0, len(s.blobs))
59 for _, b := range s.blobs {
60 out = append(out, *b)
61 }
62 sort.Slice(out, func(i, j int) bool { return out[i].Hash < out[j].Hash })
63 return out
64 }
65
66 // Backup writes a manifest and every blob as a tar stream.
67 func (s *Server) Backup(w io.Writer) error {
68 blobs := s.Snapshot()
69 manifest := BackupManifest{Version: backupVersion, Blobs: make([]BackupEntry, 0, len(blobs))}
70 for _, b := range blobs {
71 manifest.Blobs = append(manifest.Blobs, BackupEntry{
72 SHA256: b.Hash, PubKey: b.PubKey, Size: b.Size, Type: b.Type, Created: b.Created,
73 })
74 }
75 body, err := json.Marshal(manifest)
76 if err != nil {
77 return err
78 }
79 tw := tar.NewWriter(w)
80 if err := writeTarBytes(tw, backupManifest, body, time.Now().Unix()); err != nil {
81 return err
82 }
83 for _, b := range blobs {
84 path := s.blobPath(b.Hash)
85 f, err := os.Open(path)
86 if err != nil {
87 return fmt.Errorf("blossom backup: %s: %w", b.Hash, err)
88 }
89 info, err := f.Stat()
90 if err != nil {
91 f.Close()
92 return err
93 }
94 hdr := &tar.Header{
95 Name: backupBlobPrefix + b.Hash,
96 Mode: 0o644,
97 Size: info.Size(),
98 Typeflag: tar.TypeReg,
99 ModTime: time.Unix(b.Created, 0),
100 }
101 if err := tw.WriteHeader(hdr); err != nil {
102 f.Close()
103 return err
104 }
105 if _, err := io.Copy(tw, f); err != nil {
106 f.Close()
107 return err
108 }
109 if err := f.Close(); err != nil {
110 return err
111 }
112 }
113 return tw.Close()
114 }
115
116 func writeTarBytes(tw *tar.Writer, name string, data []byte, created int64) error {
117 if created == 0 {
118 created = time.Now().Unix()
119 }
120 if err := tw.WriteHeader(&tar.Header{
121 Name: name, Mode: 0o644, Size: int64(len(data)), Typeflag: tar.TypeReg,
122 ModTime: time.Unix(created, 0),
123 }); err != nil {
124 return err
125 }
126 _, err := tw.Write(data)
127 return err
128 }
129
130 // Restore reads a backup produced by Backup or the orly extractor. It verifies
131 // every SHA-256 and writes blobs into the sharded directory before adding them
132 // to the live index.
133 func (s *Server) Restore(r io.Reader) (BackupStats, error) {
134 var stats BackupStats
135 tr := tar.NewReader(r)
136 hdr, err := tr.Next()
137 if err != nil {
138 return stats, fmt.Errorf("blossom restore: missing manifest: %w", err)
139 }
140 if filepath.Clean(hdr.Name) != backupManifest {
141 return stats, fmt.Errorf("blossom restore: first entry is %q, want %q", hdr.Name, backupManifest)
142 }
143 raw, err := io.ReadAll(io.LimitReader(tr, 32<<20))
144 if err != nil {
145 return stats, err
146 }
147 var manifest BackupManifest
148 if err := json.Unmarshal(raw, &manifest); err != nil {
149 return stats, fmt.Errorf("blossom restore: manifest: %w", err)
150 }
151 if manifest.Version != backupVersion {
152 return stats, fmt.Errorf("blossom restore: unsupported version %d", manifest.Version)
153 }
154 entries := make(map[string]BackupEntry, len(manifest.Blobs))
155 for _, e := range manifest.Blobs {
156 e.SHA256 = strings.ToLower(e.SHA256)
157 if !nostr.IsHex64(e.SHA256) {
158 continue
159 }
160 entries[e.SHA256] = e
161 }
162
163 for {
164 hdr, err := tr.Next()
165 if err == io.EOF {
166 break
167 }
168 if err != nil {
169 return stats, err
170 }
171 name := filepath.ToSlash(hdr.Name)
172 if !strings.HasPrefix(name, backupBlobPrefix) {
173 continue
174 }
175 hash := strings.TrimPrefix(name, backupBlobPrefix)
176 if !nostr.IsHex64(hash) {
177 stats.Skipped++
178 continue
179 }
180 entry, ok := entries[hash]
181 if !ok {
182 stats.Skipped++
183 continue
184 }
185 size, err := s.restoreOne(tr, hash, entry)
186 if err != nil {
187 return stats, fmt.Errorf("blossom restore: %s: %w", hash, err)
188 }
189 if size < 0 {
190 stats.Skipped++
191 continue
192 }
193 stats.Restored++
194 }
195 return stats, nil
196 }
197
198 // restoreOne writes one blob. It returns -1 when the blob already exists with
199 // the same content and metadata was refreshed.
200 func (s *Server) restoreOne(r io.Reader, hash string, entry BackupEntry) (int64, error) {
201 path := s.blobPath(hash)
202 if _, err := os.Stat(path); err == nil {
203 // Content-addressed: the existing file is already the same bytes.
204 if err := s.addRestored(hash, entry); err != nil {
205 return -1, err
206 }
207 return -1, nil
208 }
209 if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
210 return -1, err
211 }
212 tmp := path + ".restore.tmp"
213 f, err := os.OpenFile(tmp, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o644)
214 if err != nil {
215 return -1, err
216 }
217 h := sha256.New()
218 n, err := io.Copy(io.MultiWriter(f, h), r)
219 if err != nil {
220 f.Close()
221 os.Remove(tmp)
222 return -1, err
223 }
224 if s.maxBlob > 0 && n > s.maxBlob {
225 f.Close()
226 os.Remove(tmp)
227 return -1, errors.New("blob exceeds configured maximum")
228 }
229 if got := hex.EncodeToString(h.Sum(nil)); got != hash {
230 f.Close()
231 os.Remove(tmp)
232 return -1, fmt.Errorf("sha256 mismatch: got %s", got)
233 }
234 if err := f.Sync(); err != nil {
235 f.Close()
236 os.Remove(tmp)
237 return -1, err
238 }
239 if err := f.Close(); err != nil {
240 os.Remove(tmp)
241 return -1, err
242 }
243 if err := os.Rename(tmp, path); err != nil {
244 os.Remove(tmp)
245 return -1, err
246 }
247 entry.Size = n
248 if err := s.addRestored(hash, entry); err != nil {
249 return -1, err
250 }
251 return n, nil
252 }
253
254 func (s *Server) addRestored(hash string, entry BackupEntry) error {
255 pub := ""
256 if nostr.IsHex64(entry.PubKey) {
257 pub = entry.PubKey
258 }
259 typ := entry.Type
260 if typ == "" {
261 typ = "application/octet-stream"
262 }
263 if len(typ) > 1024 {
264 typ = typ[:1024]
265 }
266 created := entry.Created
267 if created == 0 {
268 created = time.Now().Unix()
269 }
270 size := entry.Size
271 if st, err := os.Stat(s.blobPath(hash)); err == nil {
272 size = st.Size()
273 }
274 b := &Blob{Hash: hash, PubKey: pub, Size: size, Type: typ, Created: created}
275 s.mu.Lock()
276 s.putMem(b)
277 err := s.appendRecord('p', b)
278 s.mu.Unlock()
279 return err
280 }
281