main.go raw
1 // Command orly-blossom-extract reads orly's Badger blob metadata and writes a
2 // torly Blossom backup tar. Run it with orly stopped so Badger is not locked.
3 package main
4
5 import (
6 "archive/tar"
7 "crypto/sha256"
8 "encoding/hex"
9 "encoding/json"
10 "flag"
11 "fmt"
12 "io"
13 "mime"
14 "os"
15 "path/filepath"
16 "sort"
17 "strings"
18 "time"
19
20 "github.com/dgraph-io/badger/v4"
21 )
22
23 const metaPrefix = "blob:meta:"
24
25 type orlyMeta struct {
26 Pubkey []byte `json:"pubkey"`
27 MimeType string `json:"mime_type"`
28 Uploaded int64 `json:"uploaded"`
29 Size int64 `json:"size"`
30 Extension string `json:"extension"`
31 }
32
33 type entry struct {
34 SHA256 string `json:"sha256"`
35 PubKey string `json:"pubkey,omitempty"`
36 Size int64 `json:"size"`
37 Type string `json:"type"`
38 Created int64 `json:"created"`
39 Path string `json:"-"`
40 }
41
42 type manifest struct {
43 Version int `json:"version"`
44 Blobs []entry `json:"blobs"`
45 }
46
47 func main() {
48 dbDir := flag.String("db", "/home/mleku/.local/share/ORLY", "orly Badger data directory")
49 blobDir := flag.String("blossom", "", "blob directory (default DB/blossom)")
50 out := flag.String("out", "", "output tar")
51 flag.Parse()
52 if *out == "" {
53 fmt.Fprintln(os.Stderr, "orly-blossom-extract: --out is required")
54 os.Exit(1)
55 }
56 if *blobDir == "" {
57 *blobDir = filepath.Join(*dbDir, "blossom")
58 }
59 db, err := badger.Open(badger.DefaultOptions(*dbDir).WithLogger(nil))
60 if err != nil {
61 fmt.Fprintln(os.Stderr, "orly-blossom-extract: open Badger:", err)
62 os.Exit(1)
63 }
64 defer db.Close()
65
66 byHash := make(map[string]entry)
67 err = db.View(func(txn *badger.Txn) error {
68 it := txn.NewIterator(badger.DefaultIteratorOptions)
69 defer it.Close()
70 prefix := []byte(metaPrefix)
71 for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() {
72 key := it.Item().Key()
73 hash := string(key[len(prefix):])
74 if len(hash) != 64 {
75 continue
76 }
77 var m orlyMeta
78 if err := it.Item().Value(func(v []byte) error { return json.Unmarshal(v, &m) }); err != nil {
79 fmt.Fprintf(os.Stderr, "skip %s: %v\n", hash, err)
80 continue
81 }
82 pub := ""
83 if len(m.Pubkey) == 32 {
84 pub = hex.EncodeToString(m.Pubkey)
85 }
86 typ := m.MimeType
87 if typ == "" {
88 typ = "application/octet-stream"
89 }
90 byHash[hash] = entry{
91 SHA256: hash, PubKey: pub, Size: m.Size, Type: typ,
92 Created: m.Uploaded, Path: filepath.Join(*blobDir, hash+m.Extension),
93 }
94 }
95 return nil
96 })
97 if err != nil {
98 fmt.Fprintln(os.Stderr, "orly-blossom-extract: iterate metadata:", err)
99 os.Exit(1)
100 }
101
102 // Files without metadata still carry valid content-addressed names; include
103 // them with an empty owner and inferred type.
104 names, err := os.ReadDir(*blobDir)
105 if err != nil {
106 fmt.Fprintln(os.Stderr, "orly-blossom-extract: read blob dir:", err)
107 os.Exit(1)
108 }
109 for _, de := range names {
110 if de.IsDir() {
111 continue
112 }
113 name := de.Name()
114 if strings.Contains(name, ".tmp") || len(name) < 64 {
115 continue
116 }
117 hash := name[:64]
118 if !isHex64(hash) {
119 continue
120 }
121 if _, ok := byHash[hash]; ok {
122 continue
123 }
124 info, err := de.Info()
125 if err != nil {
126 continue
127 }
128 ext := strings.TrimPrefix(name[64:], ".")
129 typ := mime.TypeByExtension("." + ext)
130 if typ == "" {
131 typ = "application/octet-stream"
132 }
133 byHash[hash] = entry{
134 SHA256: hash, Size: info.Size(), Type: typ,
135 Created: info.ModTime().Unix(), Path: filepath.Join(*blobDir, name),
136 }
137 }
138
139 list := make([]entry, 0, len(byHash))
140 for _, e := range byHash {
141 list = append(list, e)
142 }
143 sort.Slice(list, func(i, j int) bool { return list[i].SHA256 < list[j].SHA256 })
144
145 f, err := os.Create(*out)
146 if err != nil {
147 fmt.Fprintln(os.Stderr, "orly-blossom-extract:", err)
148 os.Exit(1)
149 }
150 tw := tar.NewWriter(f)
151 if err := writeManifest(tw, list); err != nil {
152 fmt.Fprintln(os.Stderr, "orly-blossom-extract:", err)
153 os.Exit(1)
154 }
155 written, skipped := 0, 0
156 for i := range list {
157 e := &list[i]
158 body, err := os.Open(e.Path)
159 if err != nil {
160 fmt.Fprintf(os.Stderr, "skip %s: %v\n", e.SHA256, err)
161 skipped++
162 continue
163 }
164 info, err := body.Stat()
165 if err != nil {
166 body.Close()
167 skipped++
168 continue
169 }
170 h := sha256.New()
171 tmp := filepath.Join(os.TempDir(), "orly-blossom-"+e.SHA256)
172 dst, err := os.Create(tmp)
173 if err != nil {
174 body.Close()
175 fmt.Fprintln(os.Stderr, "orly-blossom-extract:", err)
176 os.Exit(1)
177 }
178 n, err := io.Copy(io.MultiWriter(dst, h), body)
179 body.Close()
180 dst.Close()
181 if err != nil || n != info.Size() || hex.EncodeToString(h.Sum(nil)) != e.SHA256 {
182 os.Remove(tmp)
183 fmt.Fprintf(os.Stderr, "skip %s: content hash/size mismatch\n", e.SHA256)
184 skipped++
185 continue
186 }
187 e.Size = n
188 if e.Created == 0 {
189 e.Created = info.ModTime().Unix()
190 }
191 // Rewrite the manifest entry now that the size is known on every
192 // restore path. The manifest already carries the original size.
193 if err := tw.WriteHeader(&tar.Header{
194 Name: "blobs/" + e.SHA256, Mode: 0o644, Size: n, Typeflag: tar.TypeReg,
195 ModTime: time.Unix(e.Created, 0),
196 }); err != nil {
197 fmt.Fprintln(os.Stderr, "orly-blossom-extract:", err)
198 os.Exit(1)
199 }
200 src, err := os.Open(tmp)
201 if err != nil {
202 fmt.Fprintln(os.Stderr, "orly-blossom-extract:", err)
203 os.Exit(1)
204 }
205 if _, err := io.Copy(tw, src); err != nil {
206 fmt.Fprintln(os.Stderr, "orly-blossom-extract:", err)
207 os.Exit(1)
208 }
209 src.Close()
210 os.Remove(tmp)
211 written++
212 }
213 if err := tw.Close(); err != nil {
214 fmt.Fprintln(os.Stderr, "orly-blossom-extract:", err)
215 os.Exit(1)
216 }
217 if err := f.Sync(); err != nil {
218 fmt.Fprintln(os.Stderr, "orly-blossom-extract:", err)
219 os.Exit(1)
220 }
221 if err := f.Close(); err != nil {
222 fmt.Fprintln(os.Stderr, "orly-blossom-extract:", err)
223 os.Exit(1)
224 }
225 fmt.Printf("{\"file\":%q,\"blobs\":%d,\"skipped\":%d,\"bytes\":%d}\n", *out, written, skipped, fileSize(*out))
226 }
227
228 func writeManifest(tw *tar.Writer, list []entry) error {
229 body, err := json.Marshal(manifest{Version: 1, Blobs: list})
230 if err != nil {
231 return err
232 }
233 if err := tw.WriteHeader(&tar.Header{
234 Name: "manifest.json", Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg,
235 ModTime: time.Now(),
236 }); err != nil {
237 return err
238 }
239 _, err = tw.Write(body)
240 return err
241 }
242
243 func isHex64(s string) bool {
244 if len(s) != 64 {
245 return false
246 }
247 for i := range s {
248 c := s[i]
249 if !(c >= '0' && c <= '9') && !(c >= 'a' && c <= 'f') {
250 return false
251 }
252 }
253 return true
254 }
255
256 func fileSize(path string) int64 {
257 info, err := os.Stat(path)
258 if err != nil {
259 return 0
260 }
261 return info.Size()
262 }
263