8356
Wx
www-xonecas-io / files
www-xonecas-io / cmd / site / main.go
5.5 KB
1// Command site serves xonecas.io from bare git repos, and manages the repos. 2// 3// site import [-sources sources.txt] [-store repos] [-jobs 8] [-meta data/meta.json] 4// site remote [-sources sources.txt] [-store repos] [-meta data/meta.json] list | new <name> | add <url or path> | rm <url or path> 5// site -c <command>, run by the login shell from make login-shell for git push, git clone and remote 6// site index [-sources sources.txt] [-store repos] [-overrides overrides.txt] [-meta data/meta.json] [-out data/projects.json] 7// site serve [-data data/projects.json] [-tailwind bin/tailwindcss] [-addr :8080] [-devnotes] 8package main 9 10import ( 11 "encoding/json" 12 "flag" 13 "fmt" 14 "log" 15 "net/http" 16 "os" 17 "path/filepath" 18 "sync" 19 20 "xonecas.io/site/internal/project" 21 "xonecas.io/site/internal/server" 22 "xonecas.io/site/internal/store" 23) 24 25func main() { 26 if len(os.Args) < 2 { 27 printUsage(os.Stderr, nil) 28 os.Exit(2) 29 } 30 fs := flag.NewFlagSet(os.Args[1], flag.ExitOnError) 31 fs.Usage = func() { printUsage(os.Stderr, fs) } 32 switch os.Args[1] { 33 case "help", "-h", "--help": 34 printUsage(os.Stdout, nil) 35 case "import": 36 sources := fs.String("sources", "sources.txt", "clone URLs, one per line") 37 root := fs.String("store", "repos", "folder for the bare mirrors") 38 jobs := fs.Int("jobs", 8, "remotes to sync at once") 39 meta := fs.String("meta", "data/meta.json", "where to save the hosts' metadata") 40 fs.Parse(os.Args[2:]) 41 urls, err := store.Sources(*sources) 42 if err != nil { 43 log.Fatal(err) 44 } 45 if err := importAll(*root, urls, urls, *jobs, *meta); err != nil { 46 log.Fatal(err) 47 } 48 case "remote": 49 sources := fs.String("sources", "sources.txt", "clone URLs, one per line") 50 root := fs.String("store", "repos", "folder for the bare mirrors") 51 meta := fs.String("meta", "data/meta.json", "where to save the hosts' metadata") 52 fs.Parse(os.Args[2:]) 53 if err := remote(fs.Args(), *sources, *root, *meta); err != nil { 54 log.Fatal(err) 55 } 56 case "-c": 57 // The login shell from make login-shell runs site -c <command> for git push, git clone and remote. 58 if len(os.Args) != 3 { 59 printUsage(os.Stderr, nil) 60 os.Exit(2) 61 } 62 if err := shell(os.Args[2]); err != nil { 63 log.Fatal(err) 64 } 65 case "index": 66 sources := fs.String("sources", "sources.txt", "clone URLs, one per line") 67 root := fs.String("store", "repos", "folder of the bare mirrors") 68 overrides := fs.String("overrides", "overrides.txt", "rules for what the heuristics get wrong") 69 meta := fs.String("meta", "data/meta.json", "the hosts' metadata, from import") 70 out := fs.String("out", "data/projects.json", "the data file") 71 fs.Parse(os.Args[2:]) 72 if err := index(*sources, *root, *overrides, *meta, *out); err != nil { 73 log.Fatal(err) 74 } 75 case "serve": 76 data := fs.String("data", "data/projects.json", "the data file, from site index; a newer one is read on the next request") 77 tailwind := fs.String("tailwind", "bin/tailwindcss", "tailwind binary that compiles site.css, empty to skip CSS") 78 addr := fs.String("addr", ":8080", "listen address, on every interface so the host can reach the VM") 79 notes := fs.Bool("devnotes", false, "add the Rats devnotes script to every page, for development") 80 fs.Parse(os.Args[2:]) 81 srv, err := server.New(*data, *tailwind) 82 if err != nil { 83 log.Fatal(err) 84 } 85 var h http.Handler = srv 86 if *notes { 87 h = devnotes(h) 88 } 89 log.Printf("serving %s on %s", *data, *addr) 90 log.Fatal(http.ListenAndServe(*addr, h)) 91 default: 92 printUsage(os.Stderr, nil) 93 os.Exit(2) 94 } 95} 96 97// mirrorAll mirrors every url into root, jobs at a time, logs each result and returns the failure count. 98func mirrorAll(root string, urls []string, jobs int) (failed int) { 99 var mu sync.Mutex 100 counts := map[string]int{} 101 work := make(chan string) 102 var wg sync.WaitGroup 103 for range jobs { 104 wg.Go(func() { 105 for url := range work { 106 state, err := store.Sync(root, url) 107 mu.Lock() 108 if err != nil { 109 failed++ 110 log.Printf("failed %s: %v", url, err) 111 } else { 112 counts[state]++ 113 log.Printf("%-7s %s", state, url) 114 } 115 mu.Unlock() 116 } 117 }) 118 } 119 for _, url := range urls { 120 work <- url 121 } 122 close(work) 123 wg.Wait() 124 log.Printf("%d new, %d updated, %d same, %d failed", counts["new"], counts["updated"], counts["same"], failed) 125 return failed 126} 127 128// index groups the mirrored remotes into projects, describes each one, and writes them to out. 129func index(sources, root, overrides, meta, out string) error { 130 urls, err := store.Sources(sources) 131 if err != nil { 132 return err 133 } 134 o, err := project.ReadOverrides(overrides) 135 if err != nil { 136 return err 137 } 138 var metas map[string]store.Meta 139 b, err := os.ReadFile(meta) 140 if err != nil { 141 return fmt.Errorf("%w: run site import first", err) 142 } 143 if err := json.Unmarshal(b, &metas); err != nil { 144 return err 145 } 146 srcs := make([]project.Source, len(urls)) 147 for i, url := range urls { 148 if srcs[i], err = project.ReadSource(url, store.Dir(root, url)); err != nil { 149 return fmt.Errorf("%s: %w", url, err) 150 } 151 } 152 projects, err := project.Group(srcs, o) 153 if err != nil { 154 return err 155 } 156 shown := 0 157 for i := range projects { 158 if err := project.Describe(&projects[i], metas, o); err != nil { 159 return fmt.Errorf("%s: %w", projects[i].Name, err) 160 } 161 if projects[i].Hidden == "" { 162 shown++ 163 } 164 } 165 log.Printf("%d remotes, %d projects, %d shown, written to %s", len(urls), len(projects), shown, out) 166 return writeJSON(out, projects) 167} 168 169func writeJSON(path string, v any) error { 170 b, err := json.MarshalIndent(v, "", " ") 171 if err != nil { 172 return err 173 } 174 if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { 175 return err 176 } 177 return os.WriteFile(path, append(b, '\n'), 0o644) 178}