xonecas.io
8356
Wx

www-xonecas-io / commit 3405bee

import every public remote into a local bare store (1003)

Co-Authored-By: Claude Opus 5.5

Sean Caetano Martin2026-11-08 3405beec04fcf23060d881fca51ff59b4acc07be parent b1edec1

7 files changed

Makefile+4−1
@@ -1,13 +1,16 @@
11 TAILWIND_VERSION := v4.3.3
22 REPOS ?= repos
33
4-.PHONY: build serve test
4+.PHONY: import build serve test
55
66 bin/tailwindcss:
77 mkdir -p bin
88 curl -fsSLo $@ https://github.com/tailwindlabs/tailwindcss/releases/download/$(TAILWIND_VERSION)/tailwindcss-linux-x64
99 chmod +x $@
1010
11+import:
12+ go run ./cmd/site import -sources sources.txt -store $(REPOS)
13+
1114 build: bin/tailwindcss
1215 go run ./cmd/site build -repos $(REPOS) -out public
1316
cmd/site/main.go+48−2
@@ -1,5 +1,6 @@
11 // Command site builds xonecas.io from bare git repos and serves it locally.
22 //
3+// site import [-sources sources.txt] [-store repos] [-jobs 8]
34 // site build [-repos repos] [-out public] [-tailwind bin/tailwindcss]
45 // site serve [-dir public] [-addr 127.0.0.1:8080]
56 package main
@@ -10,18 +11,32 @@ import (
1011 "log"
1112 "net/http"
1213 "os"
14+ "sync"
1315
1416 "xonecas.io/site/internal/repo"
1517 "xonecas.io/site/internal/site"
18+ "xonecas.io/site/internal/store"
1619 )
1720
1821 func main() {
1922 if len(os.Args) < 2 {
20- fmt.Fprintln(os.Stderr, "usage: site build|serve [flags]")
23+ fmt.Fprintln(os.Stderr, "usage: site import|build|serve [flags]")
2124 os.Exit(2)
2225 }
2326 fs := flag.NewFlagSet(os.Args[1], flag.ExitOnError)
2427 switch os.Args[1] {
28+ case "import":
29+ sources := fs.String("sources", "sources.txt", "clone URLs, one per line")
30+ root := fs.String("store", "repos", "folder for the bare mirrors")
31+ jobs := fs.Int("jobs", 8, "remotes to sync at once")
32+ fs.Parse(os.Args[2:])
33+ urls, err := store.Sources(*sources)
34+ if err != nil {
35+ log.Fatal(err)
36+ }
37+ if failed := mirrorAll(*root, urls, *jobs); failed > 0 {
38+ log.Fatalf("%d of %d remotes failed", failed, len(urls))
39+ }
2540 case "build":
2641 repos := fs.String("repos", "repos", "folder of bare *.git repos")
2742 out := fs.String("out", "public", "output folder")
@@ -42,7 +57,38 @@ func main() {
4257 log.Printf("serving %s on http://%s", *dir, *addr)
4358 log.Fatal(http.ListenAndServe(*addr, http.FileServer(http.Dir(*dir))))
4459 default:
45- fmt.Fprintln(os.Stderr, "usage: site build|serve [flags]")
60+ fmt.Fprintln(os.Stderr, "usage: site import|build|serve [flags]")
4661 os.Exit(2)
4762 }
4863 }
64+
65+// mirrorAll mirrors every url into root, jobs at a time, logs each result and returns the failure count.
66+func mirrorAll(root string, urls []string, jobs int) (failed int) {
67+ var mu sync.Mutex
68+ counts := map[string]int{}
69+ work := make(chan string)
70+ var wg sync.WaitGroup
71+ for range jobs {
72+ wg.Go(func() {
73+ for url := range work {
74+ state, err := store.Sync(root, url)
75+ mu.Lock()
76+ if err != nil {
77+ failed++
78+ log.Printf("failed %s: %v", url, err)
79+ } else {
80+ counts[state]++
81+ log.Printf("%-7s %s", state, url)
82+ }
83+ mu.Unlock()
84+ }
85+ })
86+ }
87+ for _, url := range urls {
88+ work <- url
89+ }
90+ close(work)
91+ wg.Wait()
92+ log.Printf("%d new, %d updated, %d same, %d failed", counts["new"], counts["updated"], counts["same"], failed)
93+ return failed
94+}
internal/repo/repo.go+14−10
@@ -2,6 +2,7 @@
22 package repo
33
44 import (
5+ "io/fs"
56 "os/exec"
67 "path/filepath"
78 "sort"
@@ -19,19 +20,22 @@ type Repo struct {
1920 Subject string
2021 }
2122
22-// List opens every *.git directory in dir, most recently changed first.
23+// List opens every *.git directory under dir, at any depth, most recently changed first.
2324 func List(dir string) ([]Repo, error) {
24- paths, err := filepath.Glob(filepath.Join(dir, "*.git"))
25- if err != nil {
26- return nil, err
27- }
28- repos := make([]Repo, 0, len(paths))
29- for _, p := range paths {
30- r, err := Open(p)
31- if err != nil {
32- return nil, err
25+ var repos []Repo
26+ err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
27+ if err != nil || !d.IsDir() || !strings.HasSuffix(p, ".git") {
28+ return err
3329 }
30+ r, err := Open(p)
3431 repos = append(repos, r)
32+ if err == nil {
33+ err = filepath.SkipDir
34+ }
35+ return err
36+ })
37+ if err != nil {
38+ return nil, err
3539 }
3640 sort.Slice(repos, func(i, j int) bool { return repos[i].Last.After(repos[j].Last) })
3741 return repos, nil
internal/store/store.go+83−0
@@ -0,0 +1,83 @@
1+// Package store mirrors remote git repositories into local bare repos: branches and tags only.
2+package store
3+
4+import (
5+ "bufio"
6+ "errors"
7+ "fmt"
8+ "io/fs"
9+ "os"
10+ "os/exec"
11+ "path/filepath"
12+ "strings"
13+)
14+
15+// Sources reads one clone URL per line from path. Blank lines and # comments are skipped.
16+func Sources(path string) ([]string, error) {
17+ f, err := os.Open(path)
18+ if err != nil {
19+ return nil, err
20+ }
21+ defer f.Close()
22+ var urls []string
23+ for sc := bufio.NewScanner(f); sc.Scan(); {
24+ if line := strings.TrimSpace(sc.Text()); line != "" && !strings.HasPrefix(line, "#") {
25+ urls = append(urls, line)
26+ }
27+ }
28+ return urls, err
29+}
30+
31+// Dir is where url's mirror lives under root: root/host/owner/name.git.
32+func Dir(root, url string) string {
33+ if _, rest, ok := strings.Cut(url, "://"); ok {
34+ url = rest
35+ }
36+ return filepath.Join(root, strings.TrimSuffix(url, ".git")+".git")
37+}
38+
39+// Sync creates url's mirror under root if it is missing, then fetches into it.
40+// It returns "new", "updated" or "same".
41+func Sync(root, url string) (string, error) {
42+ dir := Dir(root, url)
43+ state := "updated"
44+ if _, err := os.Stat(dir); errors.Is(err, fs.ErrNotExist) {
45+ state = "new"
46+ for _, args := range [][]string{
47+ {"init", "--bare", "--quiet", dir},
48+ {"--git-dir", dir, "remote", "add", "origin", url},
49+ {"--git-dir", dir, "config", "remote.origin.fetch", "+refs/heads/*:refs/heads/*"},
50+ {"--git-dir", dir, "config", "--add", "remote.origin.fetch", "+refs/tags/*:refs/tags/*"},
51+ } {
52+ if _, err := git(args...); err != nil {
53+ return "", err
54+ }
55+ }
56+ }
57+ before, _ := git("--git-dir", dir, "for-each-ref")
58+ if _, err := git("--git-dir", dir, "fetch", "--prune", "--quiet", "origin"); err != nil {
59+ return "", err
60+ }
61+ // HEAD follows the remote's default branch, so the site shows the branch the remote shows.
62+ if head, err := git("ls-remote", "--symref", url, "HEAD"); err == nil && strings.HasPrefix(head, "ref: ") {
63+ ref, _, _ := strings.Cut(strings.TrimPrefix(head, "ref: "), "\t")
64+ if _, err := git("--git-dir", dir, "symbolic-ref", "HEAD", ref); err != nil {
65+ return "", err
66+ }
67+ }
68+ after, err := git("--git-dir", dir, "for-each-ref")
69+ if state == "updated" && before == after {
70+ state = "same"
71+ }
72+ return state, err
73+}
74+
75+func git(args ...string) (string, error) {
76+ cmd := exec.Command("git", args...)
77+ cmd.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0")
78+ out, err := cmd.CombinedOutput()
79+ if err != nil {
80+ return "", fmt.Errorf("git %s: %w\n%s", strings.Join(args, " "), err, out)
81+ }
82+ return strings.TrimSpace(string(out)), nil
83+}
internal/store/store_test.go+55−0
@@ -0,0 +1,55 @@
1+package store
2+
3+import (
4+ "os"
5+ "os/exec"
6+ "path/filepath"
7+ "testing"
8+)
9+
10+func TestSync(t *testing.T) {
11+ work, root := t.TempDir(), t.TempDir()
12+ commit := func(msg string) {
13+ for _, args := range [][]string{{"add", "."}, {"commit", "-qm", msg}} {
14+ cmd := exec.Command("git", args...)
15+ cmd.Dir = work
16+ cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=x", "GIT_AUTHOR_EMAIL=x@x", "GIT_COMMITTER_NAME=x", "GIT_COMMITTER_EMAIL=x@x")
17+ if out, err := cmd.CombinedOutput(); err != nil {
18+ t.Fatalf("git %v: %v\n%s", args, err, out)
19+ }
20+ }
21+ }
22+ if out, err := exec.Command("git", "init", "-q", "-b", "trunk", work).CombinedOutput(); err != nil {
23+ t.Fatalf("%v\n%s", err, out)
24+ }
25+ os.WriteFile(filepath.Join(work, "a"), []byte("1"), 0o644)
26+ commit("one")
27+
28+ url := "file://" + work
29+ for i, want := range []string{"new", "same"} {
30+ if got, err := Sync(root, url); err != nil || got != want {
31+ t.Fatalf("sync %d = %q, %v; want %q", i, got, err, want)
32+ }
33+ }
34+ os.WriteFile(filepath.Join(work, "a"), []byte("2"), 0o644)
35+ commit("two")
36+ if got, err := Sync(root, url); err != nil || got != "updated" {
37+ t.Fatalf("sync after commit = %q, %v", got, err)
38+ }
39+ dir := Dir(root, url)
40+ if head, _ := git("--git-dir", dir, "symbolic-ref", "HEAD"); head != "refs/heads/trunk" {
41+ t.Errorf("HEAD = %q, want refs/heads/trunk", head)
42+ }
43+ if n, _ := git("--git-dir", dir, "rev-list", "--count", "HEAD"); n != "2" {
44+ t.Errorf("commits = %s, want 2", n)
45+ }
46+}
47+
48+func TestSources(t *testing.T) {
49+ path := filepath.Join(t.TempDir(), "sources.txt")
50+ os.WriteFile(path, []byte("# comment\n\nhttps://github.com/x/a.git\n https://gitlab.com/x/b.git \n"), 0o644)
51+ urls, err := Sources(path)
52+ if err != nil || len(urls) != 2 || Dir("r", urls[1]) != filepath.Join("r", "gitlab.com/x/b.git") {
53+ t.Fatalf("Sources = %v, %v", urls, err)
54+ }
55+}
sources.txt+106−0
@@ -0,0 +1,106 @@
1+# Every public remote the site imports, one clone URL per line. Codeberg is left out: it holds only copies.
2+https://github.com/sacenox/alfred.git
3+https://github.com/sacenox/awesome-ai-tools.git
4+https://github.com/sacenox/awesome-cli-apps.git
5+https://github.com/sacenox/bees-nest.git
6+https://github.com/sacenox/cel-tui.git
7+https://github.com/sacenox/clanker-and-the-dev.git
8+https://github.com/sacenox/client.git
9+https://github.com/sacenox/coder-skills.git
10+https://github.com/sacenox/diff-scope.git
11+https://github.com/sacenox/dotfiles.git
12+https://github.com/sacenox/githubstopannoyingme.git
13+https://github.com/sacenox/go-opencode-ai-zen-sdk.git
14+https://github.com/sacenox/hax.git
15+https://github.com/sacenox/image-pal.git
16+https://github.com/sacenox/irvc.git
17+https://github.com/sacenox/lazy-history.git
18+https://github.com/sacenox/learning-ai-lua-code-model.git
19+https://github.com/sacenox/lextide.git
20+https://github.com/sacenox/macotron-plugin-caffeinate.git
21+https://github.com/sacenox/mini-coder.git
22+https://github.com/sacenox/mini-coding-agent.git
23+https://github.com/sacenox/mysis.git
24+https://github.com/sacenox/nextjs-boilerplate.git
25+https://github.com/sacenox/nvim-bend-syntax.git
26+https://github.com/sacenox/ollama-space-molt-player.git
27+https://github.com/sacenox/orc-code.git
28+https://github.com/sacenox/PAT.git
29+https://github.com/sacenox/pi-extensions.git
30+https://github.com/sacenox/pi-linux-use.git
31+https://github.com/sacenox/pi.git
32+https://github.com/sacenox/Pickle-Plays-Space-Molt.git
33+https://github.com/sacenox/sacenox.github.io.git
34+https://github.com/sacenox/sat.git
35+https://github.com/sacenox/scrap-paper.git
36+https://github.com/sacenox/spacemolt-www.git
37+https://github.com/sacenox/stream-together.git
38+https://github.com/sacenox/symb.git
39+https://github.com/sacenox/terminals-are-sexy.git
40+https://github.com/sacenox/the-champs-timer.git
41+https://github.com/sacenox/tiny-c-agent.git
42+https://github.com/sacenox/vim-ai-complete.git
43+https://github.com/sacenox/yoctomarkdown.git
44+https://github.com/sacenox/yoctoselect.git
45+https://github.com/sacenox/Zoea-Nova.git
46+https://github.com/xonecas/2d-particle-experiements.git
47+https://github.com/xonecas/Algos.git
48+https://github.com/xonecas/BabyTheCodeShinesBright.git
49+https://github.com/xonecas/boiler.git
50+https://github.com/xonecas/bonzo.git
51+https://github.com/xonecas/BrowserQuest.git
52+https://github.com/xonecas/canvas-rings.git
53+https://github.com/xonecas/Chatroom.git
54+https://github.com/xonecas/chrome-hijs.git
55+https://github.com/xonecas/cli-node-jshint.git
56+https://github.com/xonecas/creativejs-threejs-part1.git
57+https://github.com/xonecas/disqus-node.git
58+https://github.com/xonecas/dotfiles.git
59+https://github.com/xonecas/dotter.git
60+https://github.com/xonecas/dustjs.git
61+https://github.com/xonecas/dustymarkdown.git
62+https://github.com/xonecas/ffmpeg-node.git
63+https://github.com/xonecas/firelaunch.git
64+https://github.com/xonecas/flippintables.git
65+https://github.com/xonecas/FlyingFunctions.git
66+https://github.com/xonecas/Font-Awesome.git
67+https://github.com/xonecas/fractal-tree.git
68+https://github.com/xonecas/gen-one.git
69+https://github.com/xonecas/htmlKompressor.git
70+https://github.com/xonecas/jshint.vim.git
71+https://github.com/xonecas/NameMaker.git
72+https://github.com/xonecas/night-lights.git
73+https://github.com/xonecas/NumberConversion.git
74+https://github.com/xonecas/oh-my-zsh.git
75+https://github.com/xonecas/particle-play.git
76+https://github.com/xonecas/phenny.git
77+https://github.com/xonecas/rotator.git
78+https://github.com/xonecas/shapesaver.git
79+https://github.com/xonecas/shiny-ninja-tm.git
80+https://github.com/xonecas/simple-flask-coffee-sass-boilerplate.git
81+https://github.com/xonecas/solarized.git
82+https://github.com/xonecas/startsInvasion.git
83+https://github.com/xonecas/tomorrow-theme.git
84+https://github.com/xonecas/vim-colors-256.git
85+https://github.com/xonecas/vim-getafe.git
86+https://github.com/xonecas/warez-colorscheme.git
87+https://github.com/xonecas/xonecas.com.git
88+https://github.com/xonecas/xonecas.github.com.git
89+https://github.com/xonecas/zen.git
90+https://gitlab.com/xonecas/dotfiles.git
91+https://gitlab.com/xonecas/hirenimble-challenge.git
92+https://gitlab.com/xonecas/i3wm-background-desktop-script.git
93+https://gitlab.com/xonecas/i3wm-configurations.git
94+https://gitlab.com/xonecas/jarvis.git
95+https://gitlab.com/xonecas/js-algos.git
96+https://gitlab.com/xonecas/marked-editor.git
97+https://gitlab.com/xonecas/meal_tracker.git
98+https://gitlab.com/xonecas/node-react-typescript-boilerplate.git
99+https://gitlab.com/xonecas/python-exercices.git
100+https://gitlab.com/xonecas/qr-code-maker.git
101+https://gitlab.com/xonecas/sesame-labs-coding-assignment.git
102+https://gitlab.com/xonecas/shell-greeting.git
103+https://gitlab.com/xonecas/sinfo-talk.git
104+https://gitlab.com/xonecas/tictactoe.git
105+https://gitlab.com/xonecas/vim-night-lights.git
106+https://gitlab.com/xonecas/xonecas.gitlab.io.git
tickets/1003-import-all-public-history-into-a-local-bare.md+16−3
@@ -2,13 +2,13 @@
22 id: 1003
33 title: Import all public history into a local bare repo store
44 type: feature
5-status: doing
5+status: review
66 priority: p1
77 area: data
88 created: 2026-11-07T19:30:00Z
9-updated: 2026-11-08T01:09:36Z
9+updated: 2026-11-08T01:23:43Z
1010 source: agent
11-moves: 2026-11-07T19:30:00Z inbox todo agent; 2026-11-08T01:09:36Z todo doing drag
11+moves: 2026-11-07T19:30:00Z inbox todo agent; 2026-11-08T01:09:36Z todo doing drag; 2026-11-08T01:23:43Z doing review agent
1212 ---
1313
1414 Mirror every public repo from github.com/xonecas (44), github.com/sacenox (44) and gitlab.com/xonecas (17) into bare repos, idempotently, with the source list in one file.
@@ -18,3 +18,16 @@ Codeberg stays out. Checked: 0 of its commits are missing from GitHub and GitLab
1818 Facts from discovery: plain https clones work without a token. The GitHub API allows 60 unauthenticated requests per hour, so use it for metadata only. A blobless clone of all 182 remotes was 32 MB, full clones will be larger (not measured). Forks are large, such as pi, solarized, Font-Awesome and BrowserQuest.
1919
2020 Done when: re-running fetches only what changed, and the store holds all 105 GitHub and GitLab remotes.
21+
22+## Notes
23+
24+- 2026-11-08 01:23Z (agent): Done: 'make import' mirrors all 105 public remotes into repos/ as bare repos, and a re-run only fetches what changed.
25+
26+ WHAT IS THERE:
27+ - sources.txt: the one source list, with 105 clone URLs (44 github.com/xonecas, 44 github.com/sacenox, 17 gitlab.com/xonecas). I generated it from the GitHub and GitLab APIs, 3 requests in all. Codeberg is left out.
28+ - 'site import' (internal/store): each remote goes to repos/<host>/<owner>/<name>.git, 8 at a time. The first run creates the bare repo and later runs 'git fetch --prune'. Only branches and tags are kept, so GitHub's refs/pull/* (other people's PR commits) are not fetched. HEAD follows the remote's default branch. A clone that fails is logged and makes the command exit non-zero, without stopping the others.
29+ - repos/ is gitignored. repo.List now finds *.git at any depth.
30+
31+ RESULT: 105 new, 0 failed, in 25 s. The store is 351 MB on disk (full history, forks included). The second run printed '0 new, 0 updated, 105 same, 0 failed' in 16 s.
32+ CHECKED: unit tests for new, same and updated against a local remote, the default branch, and parsing sources.txt. Spot checks: ffmpeg-node HEAD is master, mini-coder HEAD is main, and neither has pull refs.
33+ NOT CHECKED: what happens when a remote is deleted upstream. That remote would fail and be reported, and its mirror stays.