// Package walk implements an ergodic PRNG-driven repository walker. // // The walker builds a manifest of all files in a repository, // shuffles them via Fisher-Yates with a deterministic PRNG seed, // and iterates one file at a time. This ensures every file is visited // exactly once in a deterministic (but non-sequential) order. // Code, prose, config, binary data — everything gets walked. package walk import ( "io" "math" "math/rand/v2" "os" "path/filepath" "slices" "strings" "git.mleku.dev/mleku/dendrite/pkg/axiom" "git.mleku.dev/mleku/dendrite/pkg/enzyme" "git.mleku.dev/mleku/dendrite/pkg/ratio" ) // Default directories to exclude from the walk. var DefaultExclude = []string{ ".git", "node_modules", "vendor", "dist", "build", "coverage", ".svelte-kit", ".next", "memory", // live Badger database — exclude while open } // Manifest is all source files in a repository, in PRNG-permuted order. type Manifest struct { Files []string `json:"files"` // relative paths in permuted order Root string `json:"root"` Seed uint64 `json:"seed"` } // Build scans root for all files, excludes directories matching // the exclude list, and returns a manifest with files in PRNG-permuted order. func Build(root string, seed uint64, exclude []string) (*Manifest, error) { root, files, err := scanFiles(root, exclude) if err != nil { return nil, err } // Fisher-Yates shuffle with deterministic PRNG. rng := rand.New(rand.NewPCG(seed, seed^0xdeadbeef)) for i := len(files) - 1; i > 0; i-- { j := rng.IntN(i + 1) files[i], files[j] = files[j], files[i] } return &Manifest{ Files: files, Root: root, Seed: seed, }, nil } // BuildWeighted scans root for all files and orders them via // Efraimidis-Spirakis weighted random permutation. Files with higher // weight appear statistically earlier. weights maps relative file path // to weight; files not in the map get defaultWeight. func BuildWeighted(root string, seed uint64, exclude []string, weights map[string]float64, defaultWeight float64) (*Manifest, error) { root, files, err := scanFiles(root, exclude) if err != nil { return nil, err } // Efraimidis-Spirakis: for each file compute key = u^(1/w). // Sort descending by key. Higher weight → statistically higher key. rng := rand.New(rand.NewPCG(seed, seed^0xdeadbeef)) type keyed struct { path string key float64 } items := make([]keyed, len(files)) for i, f := range files { w := defaultWeight if ww, ok := weights[f]; ok { w = ww } if w < 1.0 { w = 1.0 } u := rng.Float64() for u == 0 { u = rng.Float64() } items[i] = keyed{path: f, key: math.Pow(u, 1.0/w)} } slices.SortFunc(items, func(a, b keyed) int { if a.key > b.key { return -1 } if a.key < b.key { return 1 } return 0 }) sorted := make([]string, len(items)) for i, item := range items { sorted[i] = item.path } return &Manifest{ Files: sorted, Root: root, Seed: seed, }, nil } // DeriveNextSeed produces a new seed from the previous one using SplitMix64. // Each epoch gets a different but deterministic seed. func DeriveNextSeed(prev uint64) uint64 { s := prev + 0x9e3779b97f4a7c15 s = (s ^ (s >> 30)) * 0xbf58476d1ce4e5b9 s = (s ^ (s >> 27)) * 0x94d049bb133111eb return s ^ (s >> 31) } // scanFiles walks the directory tree and returns all file relative paths. // Shared by Build and BuildWeighted. func scanFiles(root string, exclude []string) (string, []string, error) { excludeSet := make(map[string]bool, len(exclude)) for _, e := range exclude { excludeSet[e] = true } var files []string root = filepath.Clean(root) err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { if err != nil { return nil // skip unreadable entries } if d.IsDir() { if excludeSet[d.Name()] { return filepath.SkipDir } return nil } rel, err := filepath.Rel(root, path) if err != nil { return nil } files = append(files, rel) return nil }) return root, files, err } // Walker iterates through the manifest one file at a time. type Walker struct { Manifest *Manifest `json:"manifest"` Position int `json:"position"` } // NewWalker creates a walker starting at the beginning of the manifest. func NewWalker(m *Manifest) *Walker { return &Walker{Manifest: m} } // Resume creates a walker starting at the given position. func Resume(m *Manifest, position int) *Walker { return &Walker{Manifest: m, Position: position} } // Next returns the next file path (relative) and true, or ("", false) // when the walk is complete. func (w *Walker) Next() (string, bool) { if w.Position >= len(w.Manifest.Files) { return "", false } path := w.Manifest.Files[w.Position] w.Position++ return path, true } // Done reports whether all files have been visited. func (w *Walker) Done() bool { return w.Position >= len(w.Manifest.Files) } // Remaining returns the number of unvisited files. func (w *Walker) Remaining() int { r := len(w.Manifest.Files) - w.Position if r < 0 { return 0 } return r } // Progress returns the fraction of files visited as a Ratio. func (w *Walker) Progress() ratio.Ratio { total := len(w.Manifest.Files) if total == 0 { return ratio.One } return ratio.New(int64(w.Position), int64(total)) } // DigestNext opens the next file, routes it through the correct enzyme, // and returns a channel of elements. Returns nil, false when the walk // is complete. func (w *Walker) DigestNext() (<-chan axiom.Element, bool) { relPath, ok := w.Next() if !ok { return nil, false } absPath := filepath.Join(w.Manifest.Root, relPath) f, err := os.Open(absPath) if err != nil { // Skip unreadable files — return empty channel. ch := make(chan axiom.Element) close(ch) return ch, true } ch := make(chan axiom.Element, 64) go func() { defer close(ch) defer f.Close() // Emit file marker. ch <- enzyme.Elem("file", relPath) // Route to enzyme based on extension. enzCh := digestReader(absPath, f) for elem := range enzCh { ch <- elem } }() return ch, true } // digestReader selects the appropriate enzyme and digests the file. func digestReader(path string, r io.ReadSeeker) <-chan axiom.Element { ext := strings.ToLower(filepath.Ext(path)) // Read sample for enzyme detection. sample := make([]byte, 512) n, _ := r.Read(sample) sample = sample[:n] r.Seek(0, io.SeekStart) switch ext { case ".go": enz := enzyme.GoSource{} if enz.CanDigest(sample) { return enz.Digest(r) } case ".ts", ".js", ".tsx", ".jsx", ".svelte": enz := enzyme.JSSource{} if enz.CanDigest(sample) { return enz.Digest(r) } } // Fallback: text enzyme. r.Seek(0, io.SeekStart) return enzyme.Text{}.Digest(r) }