package main // git-lfs HTTP batch and download server for git.smesh.lol. // // The object store is the same tree the SSH side writes // (git-lfs-transfer, invoked through sshd-git on port 2222): // // /.git/lfs/objects/// // // so an object uploaded over SSH is immediately downloadable over HTTPS, and // nothing has to be copied between the two. // // READ ONLY, matching the git HTTP side: no repository on this host sets // http.receivepack, so HTTPS can fetch but not push. Uploads are refused here // for the same reason and `git lfs push` keeps going over SSH, where the key // in authorized_keys is what authorises it. Serving a writable LFS endpoint // over HTTPS would hand out a way to fill the disk that the git side // deliberately does not have. // // It exists because gitweb forwards /.git/* to git-http-backend, which // understands info/refs, git-upload-pack and git-receive-pack and nothing // else: /info/lfs fell through to a bare 404, so an HTTPS clone could fetch // refs and objects but never the LFS content. Caddy routes the LFS paths here. import ( "bytes" "fmt" "os" "syscall" ) // config is a sovereign self-mutating holder: a main package has no init(), so // package-level variables cannot be assigned after startup and the settings // have to live somewhere a method can write to. type config struct { repoRoot string hostURL string maxBatch int64 } func (c *config) setRoot(p string) { c.repoRoot = p } func (c *config) setHost(h string) { c.hostURL = h } func newConfig() (c *config) { return &config{ repoRoot: "/home/git", hostURL: "https://git.smesh.lol", maxBatch: 33554432, } } type request struct { method string path string hdrs map[string]string body []byte } // ── http ──────────────────────────────────────────────────────── func statusText(code int32) (s string) { switch code { case 200: return "OK" case 400: return "Bad Request" case 403: return "Forbidden" case 404: return "Not Found" case 405: return "Method Not Allowed" case 413: return "Payload Too Large" case 500: return "Internal Server Error" } return "OK" } func headerGet(req *request, key string) (v string, ok bool) { v, ok = req.hdrs[key] return v, ok } func contentLength(req *request) (n int64) { cls, ok := headerGet(req, "content-length") if !ok { return 0 } for i := 0; i < len(cls); i++ { c := cls[i] if c >= '0' && c <= '9' { n = n*10 + int64(c-'0') } } return n } func writeAll(fd int32, data []byte) (werr error) { written := int32(0) for written < len(data) { n, err := syscall.Write(fd, data[written:]) if n > 0 { written += n } if err != nil { return err } if n <= 0 { return fmt.Errorf("write stalled") } } return nil } // readRequest reads one request. Headers go into a fixed buffer and the body, // when there is one, is read as exactly Content-Length bytes. Nothing here // grows: in moxie a slice that outgrows its capacity does not realloc in place // and a subslice is a copy, so an accumulating buffer is a quadratic trap and // a large one is arena pressure. func readRequest(fd int32, maxBatch int64) (req *request, rerr error) { buf := []byte{:65536} n := int32(0) for { if n >= len(buf) { return nil, fmt.Errorf("headers too large") } rn, err := syscall.Read(fd, buf[n:]) if rn > 0 { n += rn } if bytes.Index(buf[:n], []byte("\r\n\r\n")) >= 0 { break } if err != nil || rn == 0 { if n == 0 { return nil, fmt.Errorf("empty request") } break } } sep := bytes.Index(buf[:n], []byte("\r\n\r\n")) skip := int32(4) if sep < 0 { sep = bytes.Index(buf[:n], []byte("\n\n")) skip = 2 } if sep < 0 { return nil, fmt.Errorf("no header terminator") } lines := bytes.Split(buf[:sep], []byte("\n")) if len(lines) == 0 { return nil, fmt.Errorf("empty request") } reqLine := bytes.TrimRight(lines[0], "\r") parts := bytes.SplitN(reqLine, []byte(" "), 3) if len(parts) < 2 { return nil, fmt.Errorf("bad request line") } req = &request{ method: string(parts[0]), path: string(parts[1]), hdrs: map[string]string{}, } for i := 1; i < len(lines); i++ { line := bytes.TrimRight(lines[i], "\r") if len(line) == 0 { continue } idx := bytes.IndexByte(line, ':') if idx < 0 { continue } k := bytes.ToLower(bytes.TrimSpace(line[:idx])) v := bytes.TrimSpace(line[idx+1:]) req.hdrs[string(k)] = string(v) } cl := contentLength(req) if cl > maxBatch { return req, fmt.Errorf("body too large") } if cl > 0 { bodyStart := sep + skip have := n - bodyStart if have < 0 { have = 0 } body := []byte{:int32(cl)} copied := copy(body, buf[bodyStart:n]) for copied < len(body) { rn, err := syscall.Read(fd, body[copied:]) if rn > 0 { copied += rn } if err != nil || rn == 0 { break } } if copied != len(body) { return req, fmt.Errorf("short body") } req.body = body } return req, nil } func sendHead(fd int32, code int32, ctype string, clen int64) { var out bytes.Buffer fmt.Fprintf(&out, "HTTP/1.1 %d %s\r\n", code, statusText(code)) if ctype != "" { fmt.Fprintf(&out, "Content-Type: %s\r\n", ctype) } if clen >= 0 { fmt.Fprintf(&out, "Content-Length: %d\r\n", clen) } out.WriteString("Connection: close\r\n\r\n") writeAll(fd, out.Bytes()) } func sendBody(fd int32, code int32, ctype string, body []byte) { sendHead(fd, code, ctype, int64(len(body))) writeAll(fd, body) } func sendJSON(fd int32, code int32, body []byte) { sendBody(fd, code, "application/vnd.git-lfs+json", body) } func sendError(fd int32, code int32, msg string) { var out bytes.Buffer fmt.Fprintf(&out, `{"message":%s}`, jsonQuote(msg)) out.WriteByte('\n') sendJSON(fd, code, out.Bytes()) } // ── json ──────────────────────────────────────────────────────── // // The batch body is machine-generated by git-lfs and flat, so it is scanned // rather than parsed: pulling in the decoder only to read two fields per // object would grow the build closure for no gain. func jsonQuote(s string) (q string) { var b bytes.Buffer b.WriteByte('"') for i := 0; i < len(s); i++ { c := s[i] switch c { case '"': b.WriteString("\\\"") case '\\': b.WriteString("\\\\") case '\n': b.WriteString("\\n") case '\r': b.WriteString("\\r") case '\t': b.WriteString("\\t") default: b.WriteByte(c) } } b.WriteByte('"') return b.String() } // jsonFind returns the value of the first "" at or after from, and the // offset just past it. A string value comes back unquoted, a number as its // digits. ok is false when the key is not there. func jsonFind(body []byte, key string, from int32) (val string, next int32, ok bool) { needle := []byte("\"" + key + "\"") idx := bytes.Index(body[from:], needle) if idx < 0 { return "", int32(len(body)), false } i := from + idx + int32(len(needle)) for i < int32(len(body)) && (body[i] == ' ' || body[i] == ':' || body[i] == '\t') { i++ } if i >= int32(len(body)) { return "", int32(len(body)), false } if body[i] == '"' { i++ strStart := i for i < int32(len(body)) && body[i] != '"' { if body[i] == '\\' && i+1 < int32(len(body)) { i++ } i++ } return string(body[strStart:i]), i + 1, true } numStart := i for i < int32(len(body)) && body[i] != ',' && body[i] != '}' && body[i] != ']' { i++ } return string(bytes.TrimSpace(body[numStart:i])), i, true } func jsonInt(body []byte, key string, from int32) (n int64, next int32, ok bool) { s, nx, found := jsonFind(body, key, from) if !found { return 0, nx, false } for i := 0; i < len(s); i++ { c := s[i] if c >= '0' && c <= '9' { n = n*10 + int64(c-'0') } } return n, nx, true } // ── lfs ───────────────────────────────────────────────────────── func isHex64(s string) (ok bool) { if len(s) != 64 { return false } for i := 0; i < len(s); i++ { c := s[i] if (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F') { continue } return false } return true } // safeRepo accepts only the ".git" form the URLs use, and only names made // of characters that cannot escape the repository directory. func safeRepo(urlName string) (name string, ok bool) { if len(urlName) <= 4 || urlName[len(urlName)-4:] != ".git" { return "", false } name = urlName[:len(urlName)-4] if name == "" || name == "." || name == ".." { return "", false } for i := 0; i < len(name); i++ { c := name[i] if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') { continue } switch c { case '-', '_', '.': continue } return "", false } return name, true } func objectPath(c *config, name string, oid string) (p string) { return c.repoRoot + "/" + name + ".git/lfs/objects/" + oid[:2] + "/" + oid[2:4] + "/" + oid } func objectExists(c *config, name string, oid string) (ok bool) { _, err := os.Stat(objectPath(c, name, oid)) return err == nil } func handleBatch(fd int32, c *config, req *request, name string) { if len(req.body) == 0 { sendError(fd, 400, "empty batch request") return } op, _, _ := jsonFind(req.body, "operation", 0) if op == "upload" { sendError(fd, 403, "uploads require SSH: git push over HTTPS is disabled on this server") return } if op != "download" { sendError(fd, 400, "unsupported operation") return } var out bytes.Buffer out.WriteString(`{"transfer":"basic","objects":[`) pos := int32(0) first := true for { oid, next, found := jsonFind(req.body, "oid", pos) if !found { break } pos = next size, npos, sok := jsonInt(req.body, "size", pos) if sok { pos = npos } if !isHex64(oid) { continue } if !first { out.WriteByte(',') } first = false fmt.Fprintf(&out, `{"oid":%s,"size":%d`, jsonQuote(oid), size) if objectExists(c, name, oid) { fmt.Fprintf(&out, `,"actions":{"download":{"href":%s}}`, jsonQuote(c.hostURL+"/"+name+".git/info/lfs/objects/"+oid)) } else { out.WriteString(`,"error":{"code":404,"message":"Object does not exist"}`) } out.WriteByte('}') } out.WriteString("]}\n") sendJSON(fd, 200, out.Bytes()) } func handleObject(fd int32, c *config, oid string, name string) { if !isHex64(oid) { sendError(fd, 404, "bad object id") return } p := objectPath(c, name, oid) st, serr := os.Stat(p) if serr != nil { sendError(fd, 404, "Object does not exist") return } f, err := os.Open(p) if err != nil { sendError(fd, 404, "Object does not exist") return } defer f.Close() sendHead(fd, 200, "application/octet-stream", st.Size()) buf := []byte{:1048576} for { rn, rerr := f.Read(buf) if rn > 0 { if werr := writeAll(fd, buf[:rn]); werr != nil { return } } if rn == 0 || rerr != nil { return } } } func handler(fd int32, c *config, req *request) { p := bytes.Trim([]byte(req.path), "/") parts := bytes.SplitN(p, []byte("/"), 4) if len(parts) < 4 { sendError(fd, 404, "not found") return } if string(parts[1]) != "info" || string(parts[2]) != "lfs" { sendError(fd, 404, "not found") return } name, ok := safeRepo(string(parts[0])) if !ok { sendError(fd, 404, "no such repository") return } rest := string(parts[3]) if rest == "objects/batch" { if req.method != "POST" { sendError(fd, 405, "batch requires POST") return } handleBatch(fd, c, req, name) return } if len(rest) > 8 && rest[:8] == "objects/" { if req.method != "GET" { sendError(fd, 405, "object requires GET") return } handleObject(fd, c, rest[8:], name) return } if len(rest) > 7 && rest[len(rest)-7:] == "/verify" { // Nothing to verify: an object is only ever stored by the SSH side, // which hashes what it writes before moving it into place. sendJSON(fd, 200, []byte("{}\n")) return } sendError(fd, 404, "not found") } // ── main ──────────────────────────────────────────────────────── func parseListen(addr string) (ip [4]byte, port int32) { ip = [4]byte{127, 0, 0, 1} port = 3100 i := bytes.LastIndexByte([]byte(addr), ':') if i < 0 { return ip, port } h := addr[:i] ps := addr[i+1:] if h != "" { octets := bytes.Split([]byte(h), []byte(".")) for j := 0; j < 4 && j < len(octets); j++ { v := 0 for k := 0; k < len(octets[j]); k++ { ch := octets[j][k] if ch >= '0' && ch <= '9' { v = v*10 + int32(ch-'0') } } ip[j] = byte(v) } } pv := 0 for k := 0; k < len(ps); k++ { ch := ps[k] if ch >= '0' && ch <= '9' { pv = pv*10 + int32(ch-'0') } } return ip, int32(pv) } func main() { c := newConfig() listenAddr := "127.0.0.1:3100" for i := 1; i < len(os.Args); i++ { switch os.Args[i] { case "-repos": i++ if i < len(os.Args) { c.setRoot(os.Args[i]) } case "-listen": i++ if i < len(os.Args) { listenAddr = os.Args[i] } case "-host": i++ if i < len(os.Args) { c.setHost(os.Args[i]) } } } ip, port := parseListen(listenAddr) fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_STREAM, 0) if err != nil { fmt.Fprintf(os.Stderr, "socket: %v\n", err) os.Exit(1) } syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1) sa := &syscall.SockaddrInet4{Port: int32(port), Addr: ip} if berr := syscall.Bind(fd, sa); berr != nil { fmt.Fprintf(os.Stderr, "bind: %v\n", berr) os.Exit(1) } if lerr := syscall.Listen(fd, 128); lerr != nil { fmt.Fprintf(os.Stderr, "listen: %v\n", lerr) os.Exit(1) } fmt.Printf("git-lfs-http %s repos=%s host=%s\n", listenAddr, c.repoRoot, c.hostURL) for { nfd, _, aerr := syscall.Accept(fd) if aerr != nil { continue } req, rerr := readRequest(nfd, c.maxBatch) if rerr != nil { sendError(nfd, 400, rerr.Error()) syscall.Close(nfd) continue } handler(nfd, c, req) syscall.Close(nfd) } }