xonecas.io
8058
Ir

irvc / files

irvc / preview.go

14.0 KB
1package main
2
3import (
4	"context"
5	"database/sql"
6	"encoding/json"
7	"errors"
8	"io"
9	"mime"
10	"net"
11	"net/http"
12	"net/url"
13	"os"
14	"path/filepath"
15	"regexp"
16	"strconv"
17	"strings"
18	"syscall"
19	"time"
20	"unicode"
21	"unicode/utf8"
22
23	"github.com/gin-gonic/gin"
24	"golang.org/x/net/html"
25	"golang.org/x/net/html/charset"
26)
27
28// Link previews ("cards"): the first link in a message is fetched once, by the
29// server, and the Open Graph data found in its <head> is stored with the
30// message. Clients never contact the linked site, and the thumbnail is copied
31// into our own media store so viewers do not leak their address to it either.
32
33const (
34	// maxPreviewBytes bounds how much of a page we will read looking for the
35	// <head>. Pages are read lazily and abandoned at </head>, so a link to a
36	// huge file costs at most this much, and usually nothing: non-HTML content
37	// types are rejected before the body is touched.
38	maxPreviewBytes = 4 << 20
39	maxPreviewImage = 2 << 20
40	previewTimeout  = 10 * time.Second
41	maxRedirects    = 3
42)
43
44// Preview kinds. Only "card" is rendered; the rest are recorded so that we do
45// not fetch the same link again.
46const (
47	kindCard  = "card"
48	kindImage = "image" // the link is an image
49	kindOther = "other" // not HTML (a download, a PDF, ...) or unreadable
50	kindNone  = "none"  // HTML, but without Open Graph data
51)
52
53type Preview struct {
54	URL         string `json:"url"`
55	Kind        string `json:"kind"`
56	Title       string `json:"title,omitempty"`
57	Description string `json:"description,omitempty"`
58	Image       string `json:"image,omitempty"`
59	Site        string `json:"site,omitempty"`
60}
61
62var (
63	imageMarkdown = regexp.MustCompile(`!\[[^\]]*\]\([^)]*\)`)
64	linkPattern   = regexp.MustCompile(`https?://[^\s<>]+`)
65)
66
67// firstLink returns the first http(s) URL in a message body, or "". Markdown
68// image embeds (uploads, gifs) are skipped: they are already shown inline.
69func firstLink(body string) string {
70	loc := linkPattern.FindString(imageMarkdown.ReplaceAllString(body, " "))
71	// Trailing punctuation belongs to the sentence, not the URL. A closing
72	// paren is kept only when the URL has a matching opening one.
73	for loc != "" {
74		last := loc[len(loc)-1]
75		switch {
76		case strings.IndexByte(".,;:!?'\"]}*_", last) >= 0:
77		case last == ')' && strings.Count(loc, ")") > strings.Count(loc, "("):
78		default:
79			u, err := url.Parse(loc)
80			if err != nil || u.Host == "" {
81				return ""
82			}
83			return loc
84		}
85		loc = loc[:len(loc)-1]
86	}
87	return ""
88}
89
90var errBlockedAddr = errors.New("address not allowed")
91
92// Ranges that are not covered by the net.IP predicates but are not the public
93// internet either.
94var extraBlocked = func() []*net.IPNet {
95	var out []*net.IPNet
96	for _, c := range []string{"100.64.0.0/10", "198.18.0.0/15", "192.0.0.0/24", "240.0.0.0/4", "0.0.0.0/8"} {
97		_, n, _ := net.ParseCIDR(c)
98		out = append(out, n)
99	}
100	return out
101}()
102
103// publicIP reports whether ip is somewhere a user-supplied link may send us.
104func publicIP(ip net.IP) bool {
105	if ip == nil || ip.IsLoopback() || ip.IsPrivate() || ip.IsUnspecified() ||
106		ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsMulticast() {
107		return false
108	}
109	for _, n := range extraBlocked {
110		if n.Contains(ip) {
111			return false
112		}
113	}
114	return true
115}
116
117type previewer struct {
118	client   *http.Client
119	mediaDir string
120	sem      chan struct{}
121}
122
123// newPreviewer builds the fetcher. The address check runs inside the dialer,
124// on the IP actually being connected to, so it covers redirects and DNS tricks
125// (rebinding) that checking the URL up front would miss. allowPrivate exists
126// for tests that talk to a local server.
127func newPreviewer(mediaDir string, allowPrivate bool) *previewer {
128	dialer := &net.Dialer{
129		Timeout: 5 * time.Second,
130		Control: func(_, address string, _ syscall.RawConn) error {
131			host, _, err := net.SplitHostPort(address)
132			if err != nil || (!allowPrivate && !publicIP(net.ParseIP(host))) {
133				return errBlockedAddr
134			}
135			return nil
136		},
137	}
138	return &previewer{
139		mediaDir: mediaDir,
140		sem:      make(chan struct{}, 4),
141		client: &http.Client{
142			Timeout: previewTimeout,
143			Transport: &http.Transport{
144				Proxy:                  nil, // a proxy would bypass the dial check
145				DialContext:            dialer.DialContext,
146				TLSHandshakeTimeout:    5 * time.Second,
147				ResponseHeaderTimeout:  5 * time.Second,
148				MaxResponseHeaderBytes: 64 << 10,
149				DisableKeepAlives:      true,
150			},
151			CheckRedirect: func(req *http.Request, via []*http.Request) error {
152				if len(via) > maxRedirects {
153					return errors.New("too many redirects")
154				}
155				if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
156					return errors.New("bad redirect scheme")
157				}
158				return nil
159			},
160		},
161	}
162}
163
164func (p *previewer) get(ctx context.Context, rawURL, accept string) (*http.Response, error) {
165	req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
166	if err != nil {
167		return nil, err
168	}
169	req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; irvc-link-preview)")
170	req.Header.Set("Accept", accept)
171	req.Header.Set("Accept-Language", "en")
172	resp, err := p.client.Do(req)
173	if err != nil {
174		return nil, err
175	}
176	if resp.StatusCode < 200 || resp.StatusCode > 299 {
177		resp.Body.Close()
178		return nil, errors.New("status " + resp.Status)
179	}
180	return resp, nil
181}
182
183// fetch builds the preview for link. It returns nil only when the link could
184// not be fetched at all, so a transient failure is retried on the next edit.
185func (p *previewer) fetch(ctx context.Context, link string) *Preview {
186	resp, err := p.get(ctx, link, "text/html,application/xhtml+xml;q=0.9,*/*;q=0.1")
187	if err != nil {
188		return nil
189	}
190	defer resp.Body.Close()
191	out := &Preview{URL: link, Kind: kindOther}
192
193	ctype := resp.Header.Get("Content-Type")
194	media, _, _ := mime.ParseMediaType(ctype)
195	switch {
196	case strings.HasPrefix(media, "image/"):
197		out.Kind = kindImage
198		return out
199	case media != "text/html" && media != "application/xhtml+xml":
200		return out
201	}
202
203	og := parseOpenGraph(io.LimitReader(resp.Body, maxPreviewBytes), ctype)
204	if og["title"] == "" && og["description"] == "" && og["image"] == "" {
205		out.Kind = kindNone
206		return out
207	}
208	out.Kind = kindCard
209	out.Title = clip(og["title"], 200)
210	out.Description = clip(og["description"], 300)
211	out.Site = clip(og["site_name"], 80)
212	if img := og["image"]; img != "" && resp.Request != nil {
213		if ref, err := resp.Request.URL.Parse(img); err == nil && (ref.Scheme == "http" || ref.Scheme == "https") {
214			out.Image = p.cacheImage(ctx, ref.String())
215		}
216	}
217	return out
218}
219
220// parseOpenGraph reads og:* meta tags from the document head, stopping at
221// </head> or <body> (or the end of r) rather than parsing the whole page.
222func parseOpenGraph(r io.Reader, contentType string) map[string]string {
223	og := map[string]string{}
224	if cr, err := charset.NewReader(r, contentType); err == nil {
225		r = cr
226	}
227	z := html.NewTokenizer(r)
228	for {
229		switch z.Next() {
230		case html.ErrorToken:
231			return og
232		case html.EndTagToken:
233			if name, _ := z.TagName(); string(name) == "head" {
234				return og
235			}
236		case html.StartTagToken, html.SelfClosingTagToken:
237			name, hasAttr := z.TagName()
238			switch string(name) {
239			case "body":
240				return og
241			case "meta":
242				if !hasAttr {
243					continue
244				}
245				var prop, content string
246				for {
247					k, v, more := z.TagAttr()
248					switch string(k) {
249					case "property", "name":
250						if prop == "" {
251							prop = strings.ToLower(string(v))
252						}
253					case "content":
254						content = string(v)
255					}
256					if !more {
257						break
258					}
259				}
260				if key, ok := strings.CutPrefix(prop, "og:"); ok && og[key] == "" {
261					og[key] = cleanText(content)
262				}
263			}
264		}
265	}
266}
267
268var entity = regexp.MustCompile(`&(#[0-9]+|#[xX][0-9a-fA-F]+|[A-Za-z][A-Za-z0-9]*);`)
269
270// cleanText turns a meta tag value into plain text for display. The tokenizer
271// has already decoded the attribute once (so &amp; is & and &#8217; is ’), but
272// plenty of sites encode twice (&amp;amp;), which would show up as a literal
273// "&amp;" in the card. So: decode once more if entities remain, drop control
274// characters and collapse whitespace. The result is plain text, never HTML;
275// the client renders it as a text node, so a title containing "<b>" displays as
276// those characters rather than being interpreted.
277func cleanText(s string) string {
278	s = strings.ToValidUTF8(s, "")
279	if entity.MatchString(s) {
280		s = html.UnescapeString(s)
281	}
282	s = strings.Map(func(r rune) rune {
283		if unicode.IsControl(r) && !unicode.IsSpace(r) {
284			return -1
285		}
286		return r
287	}, s)
288	return strings.Join(strings.Fields(s), " ")
289}
290
291func clip(s string, n int) string {
292	if utf8.RuneCountInString(s) <= n {
293		return s
294	}
295	r := []rune(s)
296	return strings.TrimSpace(string(r[:n-1])) + "…"
297}
298
299// cacheImage downloads a preview thumbnail into the media store and returns
300// its local URL, or "" if it is not a small raster image.
301func (p *previewer) cacheImage(ctx context.Context, src string) string {
302	resp, err := p.get(ctx, src, "image/*")
303	if err != nil {
304		return ""
305	}
306	defer resp.Body.Close()
307	data, err := io.ReadAll(io.LimitReader(resp.Body, maxPreviewImage+1))
308	if err != nil || len(data) > maxPreviewImage || len(data) == 0 {
309		return ""
310	}
311	ct := http.DetectContentType(data)
312	ext, ok := mediaExt[ct]
313	if !ok || !strings.HasPrefix(ct, "image/") {
314		return ""
315	}
316	name := randToken() + ext
317	if err := os.WriteFile(filepath.Join(p.mediaDir, name), data, 0o644); err != nil {
318		return ""
319	}
320	return "/api/media/" + name
321}
322
323func (s *Server) messagePreview(msgID int64) *Preview {
324	var raw sql.NullString
325	if s.db.QueryRow(`SELECT preview FROM messages WHERE id = ?`, msgID).Scan(&raw) != nil {
326		return nil
327	}
328	return decodePreview(raw.String)
329}
330
331func decodePreview(raw string) *Preview {
332	if raw == "" {
333		return nil
334	}
335	var p Preview
336	if json.Unmarshal([]byte(raw), &p) != nil {
337		return nil
338	}
339	return &p
340}
341
342// refreshPreview brings a message's stored preview in line with its current
343// body. It runs in the background after a message is sent or edited, and tells
344// clients (as an edit event) only when a card appears or goes away.
345func (s *Server) refreshPreview(msgID int64) {
346	s.previews.sem <- struct{}{}
347	defer func() { <-s.previews.sem }()
348
349	var body string
350	var raw sql.NullString
351	var channelID int64
352	if s.db.QueryRow(`SELECT body, preview, channel_id FROM messages WHERE id = ?`, msgID).Scan(&body, &raw, &channelID) != nil {
353		return
354	}
355	old := decodePreview(raw.String)
356	link := firstLink(body)
357
358	var next *Preview
359	if link != "" {
360		if old != nil && old.URL == link {
361			return
362		}
363		ctx, cancel := context.WithTimeout(context.Background(), previewTimeout)
364		defer cancel()
365		if next = s.previews.fetch(ctx, link); next == nil {
366			return
367		}
368	}
369
370	stored := ""
371	if next != nil {
372		b, _ := json.Marshal(next)
373		stored = string(b)
374	}
375	// Only write if the preview is still what we started from: a removal (or
376	// another refresh) that landed while we were fetching wins.
377	res, err := s.db.Exec(
378		`UPDATE messages SET preview = NULLIF(?, '') WHERE id = ? AND COALESCE(preview, '') = ?`,
379		stored, msgID, raw.String,
380	)
381	if err != nil {
382		return
383	}
384	if n, _ := res.RowsAffected(); n == 0 {
385		return
386	}
387	if (old != nil && old.Kind == kindCard) || (next != nil && next.Kind == kindCard) {
388		s.broadcastMessage(msgID, channelID)
389	}
390}
391
392// broadcastMessage re-reads a message and sends it to the channel's audience as
393// an edit, so clients replace their copy.
394func (s *Server) broadcastMessage(msgID, channelID int64) {
395	if m, chType, chName, ok := s.loadMessage(msgID, channelID); ok {
396		s.notifyChannel(chType, chName, event{Type: "message_edit", Message: &m})
397	}
398}
399
400// loadMessage reads a message as clients see it, plus its channel's type and
401// name for routing.
402func (s *Server) loadMessage(msgID, channelID int64) (m Message, chType, chName string, ok bool) {
403	var raw sql.NullString
404	err := s.db.QueryRow(
405		`SELECT m.id, m.channel_id, m.user_id, u.username, m.body, m.created_at, COALESCE(m.edited_at, 0), m.preview
406		 FROM messages m JOIN users u ON u.id = m.user_id WHERE m.id = ?`, msgID,
407	).Scan(&m.ID, &m.ChannelID, &m.UserID, &m.Username, &m.Body, &m.CreatedAt, &m.EditedAt, &raw)
408	if err != nil {
409		return m, "", "", false
410	}
411	m.Preview = decodePreview(raw.String)
412	m.Reactions = s.messageReactions(m.ID)
413	s.db.QueryRow(`SELECT type, name FROM channels WHERE id = ?`, channelID).Scan(&chType, &chName)
414	return m, chType, chName, true
415}
416
417var errNoPreview = errors.New("no preview")
418
419// removePreview deletes the card from a message its author owns. The link is
420// remembered as "none" so the same link is not fetched again on a later edit,
421// and the cached thumbnail is deleted along with it. It returns the message's
422// channel for notification.
423func (s *Server) removePreview(msgID, userID int64) (channelID int64, err error) {
424	var raw sql.NullString
425	if err := s.db.QueryRow(
426		`SELECT channel_id, preview FROM messages WHERE id = ? AND user_id = ?`, msgID, userID,
427	).Scan(&channelID, &raw); err != nil {
428		return 0, err
429	}
430	old := decodePreview(raw.String)
431	if old == nil || old.Kind != kindCard {
432		return channelID, errNoPreview
433	}
434	b, _ := json.Marshal(Preview{URL: old.URL, Kind: kindNone})
435	if _, err := s.db.Exec(`UPDATE messages SET preview = ? WHERE id = ?`, string(b), msgID); err != nil {
436		return 0, err
437	}
438	if name, ok := strings.CutPrefix(old.Image, "/api/media/"); ok && mediaName.MatchString(name) {
439		os.Remove(filepath.Join(s.cfg.MediaDir, name))
440	}
441	return channelID, nil
442}
443
444func (s *Server) handleDeletePreview(c *gin.Context) {
445	msgID, err := strconv.ParseInt(c.Param("id"), 10, 64)
446	if err != nil {
447		c.JSON(http.StatusBadRequest, gin.H{"error": "bad message"})
448		return
449	}
450	user := c.MustGet("user").(User)
451	channelID, err := s.removePreview(msgID, user.ID)
452	switch {
453	case errors.Is(err, sql.ErrNoRows):
454		c.JSON(http.StatusNotFound, gin.H{"error": "message not found"})
455		return
456	case errors.Is(err, errNoPreview):
457		// Already gone: nothing to do, and the caller's goal is met.
458	case err != nil:
459		c.JSON(http.StatusInternalServerError, gin.H{"error": "db"})
460		return
461	}
462	m, chType, chName, ok := s.loadMessage(msgID, channelID)
463	if !ok {
464		c.JSON(http.StatusInternalServerError, gin.H{"error": "db"})
465		return
466	}
467	s.notifyChannel(chType, chName, event{Type: "message_edit", Message: &m})
468	c.JSON(http.StatusOK, m)
469}