@@ -0,0 +1,468 @@
1+package main
2+
3+import (
4+ "context"
5+ "database/sql"
6+ "encoding/json"
7+ "errors"
8+ "io"
9+ "mime"
10+ "net"
11+ "net/http"
12+ "net/url"
13+ "os"
14+ "path/filepath"
15+ "regexp"
16+ "strconv"
17+ "strings"
18+ "syscall"
19+ "time"
20+ "unicode"
21+ "unicode/utf8"
22+
23+ "github.com/gin-gonic/gin"
24+ "golang.org/x/net/html"
25+ "golang.org/x/net/html/charset"
26+)
27+
28+// Link previews ("cards"): the first link in a message is fetched once, by the
29+// server, and the Open Graph data found in its <head> is stored with the
30+// message. Clients never contact the linked site, and the thumbnail is copied
31+// into our own media store so viewers do not leak their address to it either.
32+
33+const (
34+ // maxPreviewBytes bounds how much of a page we will read looking for the
35+ // <head>. Pages are read lazily and abandoned at </head>, so a link to a
36+ // huge file costs at most this much, and usually nothing: non-HTML content
37+ // types are rejected before the body is touched.
38+ maxPreviewBytes = 4 << 20
39+ maxPreviewImage = 2 << 20
40+ previewTimeout = 10 * time.Second
41+ maxRedirects = 3
42+)
43+
44+// Preview kinds. Only "card" is rendered; the rest are recorded so that we do
45+// not fetch the same link again.
46+const (
47+ kindCard = "card"
48+ kindImage = "image" // the link is an image
49+ kindOther = "other" // not HTML (a download, a PDF, ...) or unreadable
50+ kindNone = "none" // HTML, but without Open Graph data
51+)
52+
53+type Preview struct {
54+ URL string `json:"url"`
55+ Kind string `json:"kind"`
56+ Title string `json:"title,omitempty"`
57+ Description string `json:"description,omitempty"`
58+ Image string `json:"image,omitempty"`
59+ Site string `json:"site,omitempty"`
60+}
61+
62+var (
63+ imageMarkdown = regexp.MustCompile(`!\[[^\]]*\]\([^)]*\)`)
64+ linkPattern = regexp.MustCompile(`https?://[^\s<>]+`)
65+)
66+
67+// firstLink returns the first http(s) URL in a message body, or "". Markdown
68+// image embeds (uploads, gifs) are skipped: they are already shown inline.
69+func firstLink(body string) string {
70+ loc := linkPattern.FindString(imageMarkdown.ReplaceAllString(body, " "))
71+ // Trailing punctuation belongs to the sentence, not the URL. A closing
72+ // paren is kept only when the URL has a matching opening one.
73+ for loc != "" {
74+ last := loc[len(loc)-1]
75+ switch {
76+ case strings.IndexByte(".,;:!?'\"]}*_", last) >= 0:
77+ case last == ')' && strings.Count(loc, ")") > strings.Count(loc, "("):
78+ default:
79+ u, err := url.Parse(loc)
80+ if err != nil || u.Host == "" {
81+ return ""
82+ }
83+ return loc
84+ }
85+ loc = loc[:len(loc)-1]
86+ }
87+ return ""
88+}
89+
90+var errBlockedAddr = errors.New("address not allowed")
91+
92+// Ranges that are not covered by the net.IP predicates but are not the public
93+// internet either.
94+var extraBlocked = func() []*net.IPNet {
95+ var out []*net.IPNet
96+ for _, c := range []string{"100.64.0.0/10", "198.18.0.0/15", "192.0.0.0/24", "240.0.0.0/4", "0.0.0.0/8"} {
97+ _, n, _ := net.ParseCIDR(c)
98+ out = append(out, n)
99+ }
100+ return out
101+}()
102+
103+// publicIP reports whether ip is somewhere a user-supplied link may send us.
104+func publicIP(ip net.IP) bool {
105+ if ip == nil || ip.IsLoopback() || ip.IsPrivate() || ip.IsUnspecified() ||
106+ ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsMulticast() {
107+ return false
108+ }
109+ for _, n := range extraBlocked {
110+ if n.Contains(ip) {
111+ return false
112+ }
113+ }
114+ return true
115+}
116+
117+type previewer struct {
118+ client *http.Client
119+ mediaDir string
120+ sem chan struct{}
121+}
122+
123+// newPreviewer builds the fetcher. The address check runs inside the dialer,
124+// on the IP actually being connected to, so it covers redirects and DNS tricks
125+// (rebinding) that checking the URL up front would miss. allowPrivate exists
126+// for tests that talk to a local server.
127+func newPreviewer(mediaDir string, allowPrivate bool) *previewer {
128+ dialer := &net.Dialer{
129+ Timeout: 5 * time.Second,
130+ Control: func(_, address string, _ syscall.RawConn) error {
131+ host, _, err := net.SplitHostPort(address)
132+ if err != nil || (!allowPrivate && !publicIP(net.ParseIP(host))) {
133+ return errBlockedAddr
134+ }
135+ return nil
136+ },
137+ }
138+ return &previewer{
139+ mediaDir: mediaDir,
140+ sem: make(chan struct{}, 4),
141+ client: &http.Client{
142+ Timeout: previewTimeout,
143+ Transport: &http.Transport{
144+ Proxy: nil, // a proxy would bypass the dial check
145+ DialContext: dialer.DialContext,
146+ TLSHandshakeTimeout: 5 * time.Second,
147+ ResponseHeaderTimeout: 5 * time.Second,
148+ MaxResponseHeaderBytes: 64 << 10,
149+ DisableKeepAlives: true,
150+ },
151+ CheckRedirect: func(req *http.Request, via []*http.Request) error {
152+ if len(via) > maxRedirects {
153+ return errors.New("too many redirects")
154+ }
155+ if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
156+ return errors.New("bad redirect scheme")
157+ }
158+ return nil
159+ },
160+ },
161+ }
162+}
163+
164+func (p *previewer) get(ctx context.Context, rawURL, accept string) (*http.Response, error) {
165+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
166+ if err != nil {
167+ return nil, err
168+ }
169+ req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; irvc-link-preview)")
170+ req.Header.Set("Accept", accept)
171+ req.Header.Set("Accept-Language", "en")
172+ resp, err := p.client.Do(req)
173+ if err != nil {
174+ return nil, err
175+ }
176+ if resp.StatusCode < 200 || resp.StatusCode > 299 {
177+ resp.Body.Close()
178+ return nil, errors.New("status " + resp.Status)
179+ }
180+ return resp, nil
181+}
182+
183+// fetch builds the preview for link. It returns nil only when the link could
184+// not be fetched at all, so a transient failure is retried on the next edit.
185+func (p *previewer) fetch(ctx context.Context, link string) *Preview {
186+ resp, err := p.get(ctx, link, "text/html,application/xhtml+xml;q=0.9,*/*;q=0.1")
187+ if err != nil {
188+ return nil
189+ }
190+ defer resp.Body.Close()
191+ out := &Preview{URL: link, Kind: kindOther}
192+
193+ ctype := resp.Header.Get("Content-Type")
194+ media, _, _ := mime.ParseMediaType(ctype)
195+ switch {
196+ case strings.HasPrefix(media, "image/"):
197+ out.Kind = kindImage
198+ return out
199+ case media != "text/html" && media != "application/xhtml+xml":
200+ return out
201+ }
202+
203+ og := parseOpenGraph(io.LimitReader(resp.Body, maxPreviewBytes), ctype)
204+ if og["title"] == "" && og["description"] == "" && og["image"] == "" {
205+ out.Kind = kindNone
206+ return out
207+ }
208+ out.Kind = kindCard
209+ out.Title = clip(og["title"], 200)
210+ out.Description = clip(og["description"], 300)
211+ out.Site = clip(og["site_name"], 80)
212+ if img := og["image"]; img != "" && resp.Request != nil {
213+ if ref, err := resp.Request.URL.Parse(img); err == nil && (ref.Scheme == "http" || ref.Scheme == "https") {
214+ out.Image = p.cacheImage(ctx, ref.String())
215+ }
216+ }
217+ return out
218+}
219+
220+// parseOpenGraph reads og:* meta tags from the document head, stopping at
221+// </head> or <body> (or the end of r) rather than parsing the whole page.
222+func parseOpenGraph(r io.Reader, contentType string) map[string]string {
223+ og := map[string]string{}
224+ if cr, err := charset.NewReader(r, contentType); err == nil {
225+ r = cr
226+ }
227+ z := html.NewTokenizer(r)
228+ for {
229+ switch z.Next() {
230+ case html.ErrorToken:
231+ return og
232+ case html.EndTagToken:
233+ if name, _ := z.TagName(); string(name) == "head" {
234+ return og
235+ }
236+ case html.StartTagToken, html.SelfClosingTagToken:
237+ name, hasAttr := z.TagName()
238+ switch string(name) {
239+ case "body":
240+ return og
241+ case "meta":
242+ if !hasAttr {
243+ continue
244+ }
245+ var prop, content string
246+ for {
247+ k, v, more := z.TagAttr()
248+ switch string(k) {
249+ case "property", "name":
250+ if prop == "" {
251+ prop = strings.ToLower(string(v))
252+ }
253+ case "content":
254+ content = string(v)
255+ }
256+ if !more {
257+ break
258+ }
259+ }
260+ if key, ok := strings.CutPrefix(prop, "og:"); ok && og[key] == "" {
261+ og[key] = cleanText(content)
262+ }
263+ }
264+ }
265+ }
266+}
267+
268+var entity = regexp.MustCompile(`&(#[0-9]+|#[xX][0-9a-fA-F]+|[A-Za-z][A-Za-z0-9]*);`)
269+
270+// cleanText turns a meta tag value into plain text for display. The tokenizer
271+// has already decoded the attribute once (so & is & and ’ is ’), but
272+// plenty of sites encode twice (&amp;), which would show up as a literal
273+// "&" in the card. So: decode once more if entities remain, drop control
274+// characters and collapse whitespace. The result is plain text, never HTML;
275+// the client renders it as a text node, so a title containing "<b>" displays as
276+// those characters rather than being interpreted.
277+func cleanText(s string) string {
278+ s = strings.ToValidUTF8(s, "")
279+ if entity.MatchString(s) {
280+ s = html.UnescapeString(s)
281+ }
282+ s = strings.Map(func(r rune) rune {
283+ if unicode.IsControl(r) && !unicode.IsSpace(r) {
284+ return -1
285+ }
286+ return r
287+ }, s)
288+ return strings.Join(strings.Fields(s), " ")
289+}
290+
291+func clip(s string, n int) string {
292+ if utf8.RuneCountInString(s) <= n {
293+ return s
294+ }
295+ r := []rune(s)
296+ return strings.TrimSpace(string(r[:n-1])) + "…"
297+}
298+
299+// cacheImage downloads a preview thumbnail into the media store and returns
300+// its local URL, or "" if it is not a small raster image.
301+func (p *previewer) cacheImage(ctx context.Context, src string) string {
302+ resp, err := p.get(ctx, src, "image/*")
303+ if err != nil {
304+ return ""
305+ }
306+ defer resp.Body.Close()
307+ data, err := io.ReadAll(io.LimitReader(resp.Body, maxPreviewImage+1))
308+ if err != nil || len(data) > maxPreviewImage || len(data) == 0 {
309+ return ""
310+ }
311+ ext, ok := imageExt[http.DetectContentType(data)]
312+ if !ok {
313+ return ""
314+ }
315+ name := randToken() + ext
316+ if err := os.WriteFile(filepath.Join(p.mediaDir, name), data, 0o644); err != nil {
317+ return ""
318+ }
319+ return "/api/media/" + name
320+}
321+
322+func (s *Server) messagePreview(msgID int64) *Preview {
323+ var raw sql.NullString
324+ if s.db.QueryRow(`SELECT preview FROM messages WHERE id = ?`, msgID).Scan(&raw) != nil {
325+ return nil
326+ }
327+ return decodePreview(raw.String)
328+}
329+
330+func decodePreview(raw string) *Preview {
331+ if raw == "" {
332+ return nil
333+ }
334+ var p Preview
335+ if json.Unmarshal([]byte(raw), &p) != nil {
336+ return nil
337+ }
338+ return &p
339+}
340+
341+// refreshPreview brings a message's stored preview in line with its current
342+// body. It runs in the background after a message is sent or edited, and tells
343+// clients (as an edit event) only when a card appears or goes away.
344+func (s *Server) refreshPreview(msgID int64) {
345+ s.previews.sem <- struct{}{}
346+ defer func() { <-s.previews.sem }()
347+
348+ var body string
349+ var raw sql.NullString
350+ var channelID int64
351+ if s.db.QueryRow(`SELECT body, preview, channel_id FROM messages WHERE id = ?`, msgID).Scan(&body, &raw, &channelID) != nil {
352+ return
353+ }
354+ old := decodePreview(raw.String)
355+ link := firstLink(body)
356+
357+ var next *Preview
358+ if link != "" {
359+ if old != nil && old.URL == link {
360+ return
361+ }
362+ ctx, cancel := context.WithTimeout(context.Background(), previewTimeout)
363+ defer cancel()
364+ if next = s.previews.fetch(ctx, link); next == nil {
365+ return
366+ }
367+ }
368+
369+ stored := ""
370+ if next != nil {
371+ b, _ := json.Marshal(next)
372+ stored = string(b)
373+ }
374+ // Only write if the preview is still what we started from: a removal (or
375+ // another refresh) that landed while we were fetching wins.
376+ res, err := s.db.Exec(
377+ `UPDATE messages SET preview = NULLIF(?, '') WHERE id = ? AND COALESCE(preview, '') = ?`,
378+ stored, msgID, raw.String,
379+ )
380+ if err != nil {
381+ return
382+ }
383+ if n, _ := res.RowsAffected(); n == 0 {
384+ return
385+ }
386+ if (old != nil && old.Kind == kindCard) || (next != nil && next.Kind == kindCard) {
387+ s.broadcastMessage(msgID, channelID)
388+ }
389+}
390+
391+// broadcastMessage re-reads a message and sends it to the channel's audience as
392+// an edit, so clients replace their copy.
393+func (s *Server) broadcastMessage(msgID, channelID int64) {
394+ if m, chType, chName, ok := s.loadMessage(msgID, channelID); ok {
395+ s.notifyChannel(chType, chName, event{Type: "message_edit", Message: &m})
396+ }
397+}
398+
399+// loadMessage reads a message as clients see it, plus its channel's type and
400+// name for routing.
401+func (s *Server) loadMessage(msgID, channelID int64) (m Message, chType, chName string, ok bool) {
402+ var raw sql.NullString
403+ err := s.db.QueryRow(
404+ `SELECT m.id, m.channel_id, m.user_id, u.username, m.body, m.created_at, COALESCE(m.edited_at, 0), m.preview
405+ FROM messages m JOIN users u ON u.id = m.user_id WHERE m.id = ?`, msgID,
406+ ).Scan(&m.ID, &m.ChannelID, &m.UserID, &m.Username, &m.Body, &m.CreatedAt, &m.EditedAt, &raw)
407+ if err != nil {
408+ return m, "", "", false
409+ }
410+ m.Preview = decodePreview(raw.String)
411+ m.Reactions = s.messageReactions(m.ID)
412+ s.db.QueryRow(`SELECT type, name FROM channels WHERE id = ?`, channelID).Scan(&chType, &chName)
413+ return m, chType, chName, true
414+}
415+
416+var errNoPreview = errors.New("no preview")
417+
418+// removePreview deletes the card from a message its author owns. The link is
419+// remembered as "none" so the same link is not fetched again on a later edit,
420+// and the cached thumbnail is deleted along with it. It returns the message's
421+// channel for notification.
422+func (s *Server) removePreview(msgID, userID int64) (channelID int64, err error) {
423+ var raw sql.NullString
424+ if err := s.db.QueryRow(
425+ `SELECT channel_id, preview FROM messages WHERE id = ? AND user_id = ?`, msgID, userID,
426+ ).Scan(&channelID, &raw); err != nil {
427+ return 0, err
428+ }
429+ old := decodePreview(raw.String)
430+ if old == nil || old.Kind != kindCard {
431+ return channelID, errNoPreview
432+ }
433+ b, _ := json.Marshal(Preview{URL: old.URL, Kind: kindNone})
434+ if _, err := s.db.Exec(`UPDATE messages SET preview = ? WHERE id = ?`, string(b), msgID); err != nil {
435+ return 0, err
436+ }
437+ if name, ok := strings.CutPrefix(old.Image, "/api/media/"); ok && mediaName.MatchString(name) {
438+ os.Remove(filepath.Join(s.cfg.MediaDir, name))
439+ }
440+ return channelID, nil
441+}
442+
443+func (s *Server) handleDeletePreview(c *gin.Context) {
444+ msgID, err := strconv.ParseInt(c.Param("id"), 10, 64)
445+ if err != nil {
446+ c.JSON(http.StatusBadRequest, gin.H{"error": "bad message"})
447+ return
448+ }
449+ user := c.MustGet("user").(User)
450+ channelID, err := s.removePreview(msgID, user.ID)
451+ switch {
452+ case errors.Is(err, sql.ErrNoRows):
453+ c.JSON(http.StatusNotFound, gin.H{"error": "message not found"})
454+ return
455+ case errors.Is(err, errNoPreview):
456+ // Already gone: nothing to do, and the caller's goal is met.
457+ case err != nil:
458+ c.JSON(http.StatusInternalServerError, gin.H{"error": "db"})
459+ return
460+ }
461+ m, chType, chName, ok := s.loadMessage(msgID, channelID)
462+ if !ok {
463+ c.JSON(http.StatusInternalServerError, gin.H{"error": "db"})
464+ return
465+ }
466+ s.notifyChannel(chType, chName, event{Type: "message_edit", Message: &m})
467+ c.JSON(http.StatusOK, m)
468+}