@@ -1,8 +1,8 @@
11 // Command site builds xonecas.io from bare git repos and serves it locally.
22 //
3-// site import [-sources sources.txt] [-store repos] [-jobs 8]
4-// site index [-sources sources.txt] [-store repos] [-overrides overrides.txt] [-out data/projects.json]
5-// site build [-repos repos] [-out public] [-tailwind bin/tailwindcss]
3+// site import [-sources sources.txt] [-store repos] [-jobs 8] [-meta data/meta.json]
4+// site index [-sources sources.txt] [-store repos] [-overrides overrides.txt] [-meta data/meta.json] [-out data/projects.json]
5+// site build [-data data/projects.json] [-out public] [-tailwind bin/tailwindcss]
66 // site serve [-dir public] [-addr 127.0.0.1:8080]
77 package main
88
@@ -17,7 +17,6 @@ import (
1717 "sync"
1818
1919 "xonecas.io/site/internal/project"
20- "xonecas.io/site/internal/repo"
2120 "xonecas.io/site/internal/site"
2221 "xonecas.io/site/internal/store"
2322 )
@@ -33,6 +32,7 @@ func main() {
3332 sources := fs.String("sources", "sources.txt", "clone URLs, one per line")
3433 root := fs.String("store", "repos", "folder for the bare mirrors")
3534 jobs := fs.Int("jobs", 8, "remotes to sync at once")
35+ meta := fs.String("meta", "data/meta.json", "where to save the hosts' metadata")
3636 fs.Parse(os.Args[2:])
3737 urls, err := store.Sources(*sources)
3838 if err != nil {
@@ -41,28 +41,40 @@ func main() {
4141 if failed := mirrorAll(*root, urls, *jobs); failed > 0 {
4242 log.Fatalf("%d of %d remotes failed", failed, len(urls))
4343 }
44+ metas, err := store.FetchMeta(urls)
45+ if err != nil {
46+ log.Fatal(err)
47+ }
48+ if err := writeJSON(*meta, metas); err != nil {
49+ log.Fatal(err)
50+ }
4451 case "index":
4552 sources := fs.String("sources", "sources.txt", "clone URLs, one per line")
4653 root := fs.String("store", "repos", "folder of the bare mirrors")
4754 overrides := fs.String("overrides", "overrides.txt", "rules for what the heuristics get wrong")
55+ meta := fs.String("meta", "data/meta.json", "the hosts' metadata, from import")
4856 out := fs.String("out", "data/projects.json", "the data file")
4957 fs.Parse(os.Args[2:])
50- if err := index(*sources, *root, *overrides, *out); err != nil {
58+ if err := index(*sources, *root, *overrides, *meta, *out); err != nil {
5159 log.Fatal(err)
5260 }
5361 case "build":
54- repos := fs.String("repos", "repos", "folder of bare *.git repos")
62+ data := fs.String("data", "data/projects.json", "the data file, from site index")
5563 out := fs.String("out", "public", "output folder")
5664 tailwind := fs.String("tailwind", "bin/tailwindcss", "tailwind binary, empty to skip CSS")
5765 fs.Parse(os.Args[2:])
58- list, err := repo.List(*repos)
66+ var projects []project.Project
67+ b, err := os.ReadFile(*data)
68+ if err == nil {
69+ err = json.Unmarshal(b, &projects)
70+ }
5971 if err != nil {
6072 log.Fatal(err)
6173 }
62- if err := site.Build(list, *out, *tailwind); err != nil {
74+ if err := site.Build(projects, *out, *tailwind); err != nil {
6375 log.Fatal(err)
6476 }
65- log.Printf("built %d repos into %s", len(list), *out)
77+ log.Printf("built %s into %s", *data, *out)
6678 case "serve":
6779 dir := fs.String("dir", "public", "folder to serve")
6880 addr := fs.String("addr", "127.0.0.1:8080", "listen address")
@@ -106,8 +118,8 @@ func mirrorAll(root string, urls []string, jobs int) (failed int) {
106118 return failed
107119 }
108120
109-// index groups the mirrored remotes into projects and writes them to out.
110-func index(sources, root, overrides, out string) error {
121+// index groups the mirrored remotes into projects, describes each one, and writes them to out.
122+func index(sources, root, overrides, meta, out string) error {
111123 urls, err := store.Sources(sources)
112124 if err != nil {
113125 return err
@@ -116,6 +128,14 @@ func index(sources, root, overrides, out string) error {
116128 if err != nil {
117129 return err
118130 }
131+ var metas map[string]store.Meta
132+ b, err := os.ReadFile(meta)
133+ if err != nil {
134+ return fmt.Errorf("%w: run site import first", err)
135+ }
136+ if err := json.Unmarshal(b, &metas); err != nil {
137+ return err
138+ }
119139 srcs := make([]project.Source, len(urls))
120140 for i, url := range urls {
121141 if srcs[i], err = project.ReadSource(url, store.Dir(root, url)); err != nil {
@@ -126,13 +146,26 @@ func index(sources, root, overrides, out string) error {
126146 if err != nil {
127147 return err
128148 }
129- b, err := json.MarshalIndent(projects, "", " ")
149+ shown := 0
150+ for i := range projects {
151+ if err := project.Describe(&projects[i], metas, o); err != nil {
152+ return fmt.Errorf("%s: %w", projects[i].Name, err)
153+ }
154+ if projects[i].Hidden == "" {
155+ shown++
156+ }
157+ }
158+ log.Printf("%d remotes, %d projects, %d shown, written to %s", len(urls), len(projects), shown, out)
159+ return writeJSON(out, projects)
160+}
161+
162+func writeJSON(path string, v any) error {
163+ b, err := json.MarshalIndent(v, "", " ")
130164 if err != nil {
131165 return err
132166 }
133- if err := os.MkdirAll(filepath.Dir(out), 0o755); err != nil {
167+ if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
134168 return err
135169 }
136- log.Printf("%d remotes, %d projects, written to %s", len(urls), len(projects), out)
137- return os.WriteFile(out, append(b, '\n'), 0o644)
170+ return os.WriteFile(path, append(b, '\n'), 0o644)
138171 }