xonecas.io
8356
Wx

www-xonecas-io / files

www-xonecas-io / cmd / site / main.go

5.5 KB
1// Command site serves xonecas.io from bare git repos, and manages the repos.
2//
3//	site import [-sources sources.txt] [-store repos] [-jobs 8] [-meta data/meta.json]
4//	site remote [-sources sources.txt] [-store repos] [-meta data/meta.json] list | new <name> | add <url or path> | rm <url or path>
5//	site -c <command>, run by the login shell from make login-shell for git push, git clone and remote
6//	site index [-sources sources.txt] [-store repos] [-overrides overrides.txt] [-meta data/meta.json] [-out data/projects.json]
7//	site serve [-data data/projects.json] [-tailwind bin/tailwindcss] [-addr :8080] [-devnotes]
8package main
9
10import (
11	"encoding/json"
12	"flag"
13	"fmt"
14	"log"
15	"net/http"
16	"os"
17	"path/filepath"
18	"sync"
19
20	"xonecas.io/site/internal/project"
21	"xonecas.io/site/internal/server"
22	"xonecas.io/site/internal/store"
23)
24
25func main() {
26	if len(os.Args) < 2 {
27		printUsage(os.Stderr, nil)
28		os.Exit(2)
29	}
30	fs := flag.NewFlagSet(os.Args[1], flag.ExitOnError)
31	fs.Usage = func() { printUsage(os.Stderr, fs) }
32	switch os.Args[1] {
33	case "help", "-h", "--help":
34		printUsage(os.Stdout, nil)
35	case "import":
36		sources := fs.String("sources", "sources.txt", "clone URLs, one per line")
37		root := fs.String("store", "repos", "folder for the bare mirrors")
38		jobs := fs.Int("jobs", 8, "remotes to sync at once")
39		meta := fs.String("meta", "data/meta.json", "where to save the hosts' metadata")
40		fs.Parse(os.Args[2:])
41		urls, err := store.Sources(*sources)
42		if err != nil {
43			log.Fatal(err)
44		}
45		if err := importAll(*root, urls, urls, *jobs, *meta); err != nil {
46			log.Fatal(err)
47		}
48	case "remote":
49		sources := fs.String("sources", "sources.txt", "clone URLs, one per line")
50		root := fs.String("store", "repos", "folder for the bare mirrors")
51		meta := fs.String("meta", "data/meta.json", "where to save the hosts' metadata")
52		fs.Parse(os.Args[2:])
53		if err := remote(fs.Args(), *sources, *root, *meta); err != nil {
54			log.Fatal(err)
55		}
56	case "-c":
57		// The login shell from make login-shell runs site -c <command> for git push, git clone and remote.
58		if len(os.Args) != 3 {
59			printUsage(os.Stderr, nil)
60			os.Exit(2)
61		}
62		if err := shell(os.Args[2]); err != nil {
63			log.Fatal(err)
64		}
65	case "index":
66		sources := fs.String("sources", "sources.txt", "clone URLs, one per line")
67		root := fs.String("store", "repos", "folder of the bare mirrors")
68		overrides := fs.String("overrides", "overrides.txt", "rules for what the heuristics get wrong")
69		meta := fs.String("meta", "data/meta.json", "the hosts' metadata, from import")
70		out := fs.String("out", "data/projects.json", "the data file")
71		fs.Parse(os.Args[2:])
72		if err := index(*sources, *root, *overrides, *meta, *out); err != nil {
73			log.Fatal(err)
74		}
75	case "serve":
76		data := fs.String("data", "data/projects.json", "the data file, from site index; a newer one is read on the next request")
77		tailwind := fs.String("tailwind", "bin/tailwindcss", "tailwind binary that compiles site.css, empty to skip CSS")
78		addr := fs.String("addr", ":8080", "listen address, on every interface so the host can reach the VM")
79		notes := fs.Bool("devnotes", false, "add the Rats devnotes script to every page, for development")
80		fs.Parse(os.Args[2:])
81		srv, err := server.New(*data, *tailwind)
82		if err != nil {
83			log.Fatal(err)
84		}
85		var h http.Handler = srv
86		if *notes {
87			h = devnotes(h)
88		}
89		log.Printf("serving %s on %s", *data, *addr)
90		log.Fatal(http.ListenAndServe(*addr, h))
91	default:
92		printUsage(os.Stderr, nil)
93		os.Exit(2)
94	}
95}
96
97// mirrorAll mirrors every url into root, jobs at a time, logs each result and returns the failure count.
98func mirrorAll(root string, urls []string, jobs int) (failed int) {
99	var mu sync.Mutex
100	counts := map[string]int{}
101	work := make(chan string)
102	var wg sync.WaitGroup
103	for range jobs {
104		wg.Go(func() {
105			for url := range work {
106				state, err := store.Sync(root, url)
107				mu.Lock()
108				if err != nil {
109					failed++
110					log.Printf("failed  %s: %v", url, err)
111				} else {
112					counts[state]++
113					log.Printf("%-7s %s", state, url)
114				}
115				mu.Unlock()
116			}
117		})
118	}
119	for _, url := range urls {
120		work <- url
121	}
122	close(work)
123	wg.Wait()
124	log.Printf("%d new, %d updated, %d same, %d failed", counts["new"], counts["updated"], counts["same"], failed)
125	return failed
126}
127
128// index groups the mirrored remotes into projects, describes each one, and writes them to out.
129func index(sources, root, overrides, meta, out string) error {
130	urls, err := store.Sources(sources)
131	if err != nil {
132		return err
133	}
134	o, err := project.ReadOverrides(overrides)
135	if err != nil {
136		return err
137	}
138	var metas map[string]store.Meta
139	b, err := os.ReadFile(meta)
140	if err != nil {
141		return fmt.Errorf("%w: run site import first", err)
142	}
143	if err := json.Unmarshal(b, &metas); err != nil {
144		return err
145	}
146	srcs := make([]project.Source, len(urls))
147	for i, url := range urls {
148		if srcs[i], err = project.ReadSource(url, store.Dir(root, url)); err != nil {
149			return fmt.Errorf("%s: %w", url, err)
150		}
151	}
152	projects, err := project.Group(srcs, o)
153	if err != nil {
154		return err
155	}
156	shown := 0
157	for i := range projects {
158		if err := project.Describe(&projects[i], metas, o); err != nil {
159			return fmt.Errorf("%s: %w", projects[i].Name, err)
160		}
161		if projects[i].Hidden == "" {
162			shown++
163		}
164	}
165	log.Printf("%d remotes, %d projects, %d shown, written to %s", len(urls), len(projects), shown, out)
166	return writeJSON(out, projects)
167}
168
169func writeJSON(path string, v any) error {
170	b, err := json.MarshalIndent(v, "", "  ")
171	if err != nil {
172		return err
173	}
174	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
175		return err
176	}
177	return os.WriteFile(path, append(b, '\n'), 0o644)
178}