xonecas.io
8356
Wx

www-xonecas-io / commit 7414053

site remote list|add|rm edits sources.txt and mirrors the new remote (1042)

Co-Authored-By: Claude Opus 5.5

Sean Caetano Martin2026-11-08 741405366758ae98f69a8093120947a062f09153 parent e623581

8 files changed

Makefile+5−1
@@ -1,13 +1,17 @@
11 TAILWIND_VERSION := v4.3.3
22 REPOS ?= repos
33
4-.PHONY: import index build serve test
4+.PHONY: site import index build serve test
55
66 bin/tailwindcss:
77 mkdir -p bin
88 curl -fsSLo $@ https://github.com/tailwindlabs/tailwindcss/releases/download/$(TAILWIND_VERSION)/tailwindcss-linux-x64
99 chmod +x $@
1010
11+# site builds the command, so bin/site remote add <url> and the rest work from here.
12+site:
13+ go build -o bin/site ./cmd/site
14+
1115 import:
1216 go run ./cmd/site import -sources sources.txt -store $(REPOS)
1317
cmd/site/main.go+10−8
@@ -1,6 +1,7 @@
11 // Command site builds xonecas.io from bare git repos and serves it locally.
22 //
33 // site import [-sources sources.txt] [-store repos] [-jobs 8] [-meta data/meta.json]
4+// site remote [-sources sources.txt] [-store repos] [-meta data/meta.json] list | add <url> | rm <url>
45 // site index [-sources sources.txt] [-store repos] [-overrides overrides.txt] [-meta data/meta.json] [-out data/projects.json]
56 // site build [-data data/projects.json] [-out public] [-tailwind bin/tailwindcss]
67 // site serve [-dir public] [-addr :8080]
@@ -23,7 +24,7 @@ import (
2324
2425 func main() {
2526 if len(os.Args) < 2 {
26- fmt.Fprintln(os.Stderr, "usage: site import|build|serve [flags]")
27+ fmt.Fprintln(os.Stderr, "usage: site import|remote|index|build|serve [flags]")
2728 os.Exit(2)
2829 }
2930 fs := flag.NewFlagSet(os.Args[1], flag.ExitOnError)
@@ -38,14 +39,15 @@ func main() {
3839 if err != nil {
3940 log.Fatal(err)
4041 }
41- if failed := mirrorAll(*root, urls, *jobs); failed > 0 {
42- log.Fatalf("%d of %d remotes failed", failed, len(urls))
43- }
44- metas, err := store.FetchMeta(urls)
45- if err != nil {
42+ if err := importAll(*root, urls, urls, *jobs, *meta); err != nil {
4643 log.Fatal(err)
4744 }
48- if err := writeJSON(*meta, metas); err != nil {
45+ case "remote":
46+ sources := fs.String("sources", "sources.txt", "clone URLs, one per line")
47+ root := fs.String("store", "repos", "folder for the bare mirrors")
48+ meta := fs.String("meta", "data/meta.json", "where to save the hosts' metadata")
49+ fs.Parse(os.Args[2:])
50+ if err := remote(fs.Args(), *sources, *root, *meta); err != nil {
4951 log.Fatal(err)
5052 }
5153 case "index":
@@ -82,7 +84,7 @@ func main() {
8284 log.Printf("serving %s on %s", *dir, *addr)
8385 log.Fatal(http.ListenAndServe(*addr, devnotes(*dir, http.FileServer(http.Dir(*dir)))))
8486 default:
85- fmt.Fprintln(os.Stderr, "usage: site import|build|serve [flags]")
87+ fmt.Fprintln(os.Stderr, "usage: site import|remote|index|build|serve [flags]")
8688 os.Exit(2)
8789 }
8890 }
cmd/site/remote.go+60−0
@@ -0,0 +1,60 @@
1+package main
2+
3+import (
4+ "errors"
5+ "fmt"
6+ "log"
7+ "os"
8+
9+ "xonecas.io/site/internal/store"
10+)
11+
12+// remote lists the remotes in the sources file, or adds or removes one.
13+// add mirrors the new remote and refreshes the hosts' metadata, as import does; rm deletes its mirror.
14+// Either way, site index and site build then show the change.
15+func remote(args []string, sources, root, meta string) error {
16+ urls, err := store.Sources(sources)
17+ if err != nil {
18+ return err
19+ }
20+ if len(args) == 1 && args[0] == "list" {
21+ for _, url := range urls {
22+ fmt.Println(url)
23+ }
24+ return nil
25+ }
26+ if len(args) != 2 || (args[0] != "add" && args[0] != "rm") {
27+ return errors.New("usage: site remote list | add <url> | rm <url>")
28+ }
29+ url, err := store.Clean(args[1])
30+ if err != nil {
31+ return err
32+ }
33+ if args[0] == "rm" {
34+ if err := store.RemoveSource(sources, url); err != nil {
35+ return err
36+ }
37+ log.Printf("removed %s; site index and site build drop it", url)
38+ return os.RemoveAll(store.Dir(root, url))
39+ }
40+ if err := store.AddSource(sources, url); err != nil {
41+ return err
42+ }
43+ if err := importAll(root, []string{url}, append(urls, url), 1, meta); err != nil {
44+ return errors.Join(err, store.RemoveSource(sources, url), os.RemoveAll(store.Dir(root, url)))
45+ }
46+ log.Printf("added %s; site index and site build list it", url)
47+ return nil
48+}
49+
50+// importAll mirrors the remotes in sync, then saves the hosts' metadata for all of them to meta.
51+func importAll(root string, sync, all []string, jobs int, meta string) error {
52+ if failed := mirrorAll(root, sync, jobs); failed > 0 {
53+ return fmt.Errorf("%d of %d remotes failed", failed, len(sync))
54+ }
55+ metas, err := store.FetchMeta(all)
56+ if err != nil {
57+ return err
58+ }
59+ return writeJSON(meta, metas)
60+}
internal/store/meta.go+9−4
@@ -25,10 +25,7 @@ func FetchMeta(urls []string) (map[string]Meta, error) {
2525 metas := map[string]Meta{}
2626 for o := range owners {
2727 host, owner, _ := strings.Cut(o, "/")
28- api := map[string]string{
29- "github.com": "https://api.github.com/users/" + owner + "/repos?per_page=100&type=owner",
30- "gitlab.com": "https://gitlab.com/api/v4/users/" + owner + "/projects?per_page=100",
31- }[host]
28+ api := ownerAPI(host, owner)
3229 if api == "" {
3330 return nil, fmt.Errorf("no API for %s", host)
3431 }
@@ -57,3 +54,11 @@ func FetchMeta(urls []string) (map[string]Meta, error) {
5754 }
5855 return metas, nil
5956 }
57+
58+// ownerAPI is the host's URL for owner's repo list, or "" for a host without one.
59+func ownerAPI(host, owner string) string {
60+ return map[string]string{
61+ "github.com": "https://api.github.com/users/" + owner + "/repos?per_page=100&type=owner",
62+ "gitlab.com": "https://gitlab.com/api/v4/users/" + owner + "/projects?per_page=100",
63+ }[host]
64+}
internal/store/sources.go+72−0
@@ -0,0 +1,72 @@
1+package store
2+
3+import (
4+ "bufio"
5+ "errors"
6+ "fmt"
7+ "os"
8+ "slices"
9+ "strings"
10+)
11+
12+// Sources reads one clone URL per line from path. Blank lines and # comments are skipped.
13+func Sources(path string) ([]string, error) {
14+ f, err := os.Open(path)
15+ if err != nil {
16+ return nil, err
17+ }
18+ defer f.Close()
19+ var urls []string
20+ for sc := bufio.NewScanner(f); sc.Scan(); {
21+ if line := strings.TrimSpace(sc.Text()); line != "" && !strings.HasPrefix(line, "#") {
22+ urls = append(urls, line)
23+ }
24+ }
25+ return urls, err
26+}
27+
28+// Clean checks that url is an https clone URL on a host import can describe, and returns it ending in .git.
29+func Clean(url string) (string, error) {
30+ rest, ok := strings.CutPrefix(url, "https://")
31+ host, path, _ := strings.Cut(rest, "/")
32+ owner, name, _ := strings.Cut(strings.TrimSuffix(strings.TrimSuffix(path, "/"), ".git"), "/")
33+ if !ok || owner == "" || name == "" || strings.Contains(name, "/") {
34+ return "", fmt.Errorf("%s is not a clone URL like https://github.com/owner/name.git", url)
35+ }
36+ if ownerAPI(host, owner) == "" {
37+ return "", fmt.Errorf("%s: import can only describe github.com and gitlab.com remotes", url)
38+ }
39+ return "https://" + host + "/" + owner + "/" + name + ".git", nil
40+}
41+
42+// AddSource appends url to the sources file at path, unless it is there already.
43+func AddSource(path, url string) error {
44+ urls, err := Sources(path)
45+ if err != nil {
46+ return err
47+ }
48+ if slices.Contains(urls, url) {
49+ return fmt.Errorf("%s is already in %s", url, path)
50+ }
51+ f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o644)
52+ if err != nil {
53+ return err
54+ }
55+ _, err = fmt.Fprintln(f, url)
56+ return errors.Join(err, f.Close())
57+}
58+
59+// RemoveSource drops url's line from the sources file at path, keeping every other line.
60+func RemoveSource(path, url string) error {
61+ b, err := os.ReadFile(path)
62+ if err != nil {
63+ return err
64+ }
65+ lines := strings.SplitAfter(string(b), "\n")
66+ n := len(lines)
67+ lines = slices.DeleteFunc(lines, func(l string) bool { return strings.TrimSpace(l) == url })
68+ if len(lines) == n {
69+ return fmt.Errorf("%s is not in %s", url, path)
70+ }
71+ return os.WriteFile(path, []byte(strings.Join(lines, "")), 0o644)
72+}
internal/store/store.go+0−17
@@ -2,7 +2,6 @@
22 package store
33
44 import (
5- "bufio"
65 "errors"
76 "fmt"
87 "io/fs"
@@ -12,22 +11,6 @@ import (
1211 "strings"
1312 )
1413
15-// Sources reads one clone URL per line from path. Blank lines and # comments are skipped.
16-func Sources(path string) ([]string, error) {
17- f, err := os.Open(path)
18- if err != nil {
19- return nil, err
20- }
21- defer f.Close()
22- var urls []string
23- for sc := bufio.NewScanner(f); sc.Scan(); {
24- if line := strings.TrimSpace(sc.Text()); line != "" && !strings.HasPrefix(line, "#") {
25- urls = append(urls, line)
26- }
27- }
28- return urls, err
29-}
30-
3114 // Dir is where url's mirror lives under root: root/host/owner/name.git.
3215 func Dir(root, url string) string {
3316 if _, rest, ok := strings.Cut(url, "://"); ok {
internal/store/store_test.go+22−0
@@ -53,3 +53,25 @@ func TestSources(t *testing.T) {
5353 t.Fatalf("Sources = %v, %v", urls, err)
5454 }
5555 }
56+
57+func TestEditSources(t *testing.T) {
58+ for in, want := range map[string]string{
59+ "https://github.com/x/a": "https://github.com/x/a.git", "https://gitlab.com/x/b.git/": "https://gitlab.com/x/b.git",
60+ "https://codeberg.org/x/c.git": "", "git@github.com:x/a.git": "", "https://github.com/x": "", "https://gitlab.com/g/sub/d.git": "",
61+ } {
62+ if got, _ := Clean(in); got != want {
63+ t.Errorf("Clean(%q) = %q, want %q", in, got, want)
64+ }
65+ }
66+ path := filepath.Join(t.TempDir(), "sources.txt")
67+ os.WriteFile(path, []byte("# comment\nhttps://github.com/x/a.git\n"), 0o644)
68+ if AddSource(path, "https://github.com/x/b.git") != nil || AddSource(path, "https://github.com/x/a.git") == nil {
69+ t.Error("AddSource adds a new remote once")
70+ }
71+ if RemoveSource(path, "https://github.com/x/a.git") != nil || RemoveSource(path, "https://github.com/x/a.git") == nil {
72+ t.Error("RemoveSource removes a listed remote once")
73+ }
74+ if b, _ := os.ReadFile(path); string(b) != "# comment\nhttps://github.com/x/b.git\n" {
75+ t.Errorf("sources file is %q", b)
76+ }
77+}
tickets/1042-how-do-we-add-new-remotes.md+14−3
@@ -1,16 +1,17 @@
11 ---
22 id: 1042
33 title: how do we add new remotes
4-type: chore
4+type: feature
55 status: review
66 priority: p2
77 created: 2026-11-08T06:26:18Z
8-updated: 2026-11-08T06:27:05Z
8+updated: 2026-11-08T07:00:33Z
99 source: human
10-moves: 2026-11-08T06:26:22Z inbox doing drag; 2026-11-08T06:27:05Z doing review agent
10+moves: 2026-11-08T06:26:22Z inbox doing drag; 2026-11-08T06:27:05Z doing review agent; 2026-11-08T06:57:08Z review doing drag; 2026-11-08T07:00:33Z doing review agent
1111 area: import
1212 ---
1313
14+Can we make this more formal, like a cli with the server bin? We could build on it going forward, like allowing me to manage pins, private/public toggle, etc.
1415
1516 ## Notes
1617
@@ -23,3 +24,13 @@ area: import
2324 - Each owner's API list is read with per_page=100 and no paging. That's fine today (44 sacenox, 44 xonecas, 17 gitlab), but an owner with more than 100 repos would get no description for the rest.
2425 If the same project lives on two hosts, add 'merge <url> <url>' to overrides.txt; hide, rename and tag rules go there too.
2526 Nothing was changed: I read the Makefile, cmd/site/main.go and internal/store.
27+
28+- 2026-11-08 07:00Z (agent): Done: the site binary has a remote command. sources.txt stays the list it edits.
29+ make site builds bin/site
30+ bin/site remote list the 105 remotes
31+ bin/site remote add <url> checks the URL, appends it, mirrors it into repos/ and refreshes data/meta.json
32+ bin/site remote rm <url> drops its line (comments kept) and deletes its mirror
33+ Then make index build (or make index serve) shows the change, the same as after make import. add takes github.com/gitlab.com https URLs with or without .git and normalises them to .git. It refuses a duplicate, another host or an SSH URL. If the mirror or metadata fails, the line and the mirror are removed again, so sources.txt stays as it was.
34+ Code: cmd/site/remote.go (the command, and importAll, which import now shares); internal/store/sources.go (Sources moved there from store.go, plus Clean, AddSource and RemoveSource); meta.go's host table became ownerAPI(), which Clean uses too. To grow it later, the next commands (pin, public/private) can sit beside remote in cmd/site and edit overrides.txt the same way.
35+ Checked: go vet and go test (new TestEditSources); a real add then rm of github.com/octocat/Hello-World on scratch copies of sources.txt, repos/ and meta.json (mirrored, description fetched, sources.txt back byte-identical after rm); a failed add of a missing repo leaves sources.txt unchanged and no mirror; the error messages for usage, codeberg and a duplicate. Not run: a full make import against all 105 remotes. Its path is the same importAll that the add test ran. Your real sources.txt, rep …
36+ Correction to my first note: an owner with 100 or more repos doesn't silently lose descriptions; FetchMeta stops with "100 repos, the list needs paging".