// Command orly-blossom-extract reads orly's Badger blob metadata and writes a // torly Blossom backup tar. Run it with orly stopped so Badger is not locked. package main import ( "archive/tar" "crypto/sha256" "encoding/hex" "encoding/json" "flag" "fmt" "io" "mime" "os" "path/filepath" "sort" "strings" "time" "github.com/dgraph-io/badger/v4" ) const metaPrefix = "blob:meta:" type orlyMeta struct { Pubkey []byte `json:"pubkey"` MimeType string `json:"mime_type"` Uploaded int64 `json:"uploaded"` Size int64 `json:"size"` Extension string `json:"extension"` } type entry struct { SHA256 string `json:"sha256"` PubKey string `json:"pubkey,omitempty"` Size int64 `json:"size"` Type string `json:"type"` Created int64 `json:"created"` Path string `json:"-"` } type manifest struct { Version int `json:"version"` Blobs []entry `json:"blobs"` } func main() { dbDir := flag.String("db", "/home/mleku/.local/share/ORLY", "orly Badger data directory") blobDir := flag.String("blossom", "", "blob directory (default DB/blossom)") out := flag.String("out", "", "output tar") flag.Parse() if *out == "" { fmt.Fprintln(os.Stderr, "orly-blossom-extract: --out is required") os.Exit(1) } if *blobDir == "" { *blobDir = filepath.Join(*dbDir, "blossom") } db, err := badger.Open(badger.DefaultOptions(*dbDir).WithLogger(nil)) if err != nil { fmt.Fprintln(os.Stderr, "orly-blossom-extract: open Badger:", err) os.Exit(1) } defer db.Close() byHash := make(map[string]entry) err = db.View(func(txn *badger.Txn) error { it := txn.NewIterator(badger.DefaultIteratorOptions) defer it.Close() prefix := []byte(metaPrefix) for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { key := it.Item().Key() hash := string(key[len(prefix):]) if len(hash) != 64 { continue } var m orlyMeta if err := it.Item().Value(func(v []byte) error { return json.Unmarshal(v, &m) }); err != nil { fmt.Fprintf(os.Stderr, "skip %s: %v\n", hash, err) continue } pub := "" if len(m.Pubkey) == 32 { pub = hex.EncodeToString(m.Pubkey) } typ := m.MimeType if typ == "" { typ = "application/octet-stream" } byHash[hash] = entry{ SHA256: hash, PubKey: pub, Size: m.Size, Type: typ, Created: m.Uploaded, Path: filepath.Join(*blobDir, hash+m.Extension), } } return nil }) if err != nil { fmt.Fprintln(os.Stderr, "orly-blossom-extract: iterate metadata:", err) os.Exit(1) } // Files without metadata still carry valid content-addressed names; include // them with an empty owner and inferred type. names, err := os.ReadDir(*blobDir) if err != nil { fmt.Fprintln(os.Stderr, "orly-blossom-extract: read blob dir:", err) os.Exit(1) } for _, de := range names { if de.IsDir() { continue } name := de.Name() if strings.Contains(name, ".tmp") || len(name) < 64 { continue } hash := name[:64] if !isHex64(hash) { continue } if _, ok := byHash[hash]; ok { continue } info, err := de.Info() if err != nil { continue } ext := strings.TrimPrefix(name[64:], ".") typ := mime.TypeByExtension("." + ext) if typ == "" { typ = "application/octet-stream" } byHash[hash] = entry{ SHA256: hash, Size: info.Size(), Type: typ, Created: info.ModTime().Unix(), Path: filepath.Join(*blobDir, name), } } list := make([]entry, 0, len(byHash)) for _, e := range byHash { list = append(list, e) } sort.Slice(list, func(i, j int) bool { return list[i].SHA256 < list[j].SHA256 }) f, err := os.Create(*out) if err != nil { fmt.Fprintln(os.Stderr, "orly-blossom-extract:", err) os.Exit(1) } tw := tar.NewWriter(f) if err := writeManifest(tw, list); err != nil { fmt.Fprintln(os.Stderr, "orly-blossom-extract:", err) os.Exit(1) } written, skipped := 0, 0 for i := range list { e := &list[i] body, err := os.Open(e.Path) if err != nil { fmt.Fprintf(os.Stderr, "skip %s: %v\n", e.SHA256, err) skipped++ continue } info, err := body.Stat() if err != nil { body.Close() skipped++ continue } h := sha256.New() tmp := filepath.Join(os.TempDir(), "orly-blossom-"+e.SHA256) dst, err := os.Create(tmp) if err != nil { body.Close() fmt.Fprintln(os.Stderr, "orly-blossom-extract:", err) os.Exit(1) } n, err := io.Copy(io.MultiWriter(dst, h), body) body.Close() dst.Close() if err != nil || n != info.Size() || hex.EncodeToString(h.Sum(nil)) != e.SHA256 { os.Remove(tmp) fmt.Fprintf(os.Stderr, "skip %s: content hash/size mismatch\n", e.SHA256) skipped++ continue } e.Size = n if e.Created == 0 { e.Created = info.ModTime().Unix() } // Rewrite the manifest entry now that the size is known on every // restore path. The manifest already carries the original size. if err := tw.WriteHeader(&tar.Header{ Name: "blobs/" + e.SHA256, Mode: 0o644, Size: n, Typeflag: tar.TypeReg, ModTime: time.Unix(e.Created, 0), }); err != nil { fmt.Fprintln(os.Stderr, "orly-blossom-extract:", err) os.Exit(1) } src, err := os.Open(tmp) if err != nil { fmt.Fprintln(os.Stderr, "orly-blossom-extract:", err) os.Exit(1) } if _, err := io.Copy(tw, src); err != nil { fmt.Fprintln(os.Stderr, "orly-blossom-extract:", err) os.Exit(1) } src.Close() os.Remove(tmp) written++ } if err := tw.Close(); err != nil { fmt.Fprintln(os.Stderr, "orly-blossom-extract:", err) os.Exit(1) } if err := f.Sync(); err != nil { fmt.Fprintln(os.Stderr, "orly-blossom-extract:", err) os.Exit(1) } if err := f.Close(); err != nil { fmt.Fprintln(os.Stderr, "orly-blossom-extract:", err) os.Exit(1) } fmt.Printf("{\"file\":%q,\"blobs\":%d,\"skipped\":%d,\"bytes\":%d}\n", *out, written, skipped, fileSize(*out)) } func writeManifest(tw *tar.Writer, list []entry) error { body, err := json.Marshal(manifest{Version: 1, Blobs: list}) if err != nil { return err } if err := tw.WriteHeader(&tar.Header{ Name: "manifest.json", Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg, ModTime: time.Now(), }); err != nil { return err } _, err = tw.Write(body) return err } func isHex64(s string) bool { if len(s) != 64 { return false } for i := range s { c := s[i] if !(c >= '0' && c <= '9') && !(c >= 'a' && c <= 'f') { return false } } return true } func fileSize(path string) int64 { info, err := os.Stat(path) if err != nil { return 0 } return info.Size() }