8058
Ir
irvc / files
irvc / preview.go
14.0 KB
1package main 2 3import ( 4 "context" 5 "database/sql" 6 "encoding/json" 7 "errors" 8 "io" 9 "mime" 10 "net" 11 "net/http" 12 "net/url" 13 "os" 14 "path/filepath" 15 "regexp" 16 "strconv" 17 "strings" 18 "syscall" 19 "time" 20 "unicode" 21 "unicode/utf8" 22 23 "github.com/gin-gonic/gin" 24 "golang.org/x/net/html" 25 "golang.org/x/net/html/charset" 26) 27 28// Link previews ("cards"): the first link in a message is fetched once, by the 29// server, and the Open Graph data found in its <head> is stored with the 30// message. Clients never contact the linked site, and the thumbnail is copied 31// into our own media store so viewers do not leak their address to it either. 32 33const ( 34 // maxPreviewBytes bounds how much of a page we will read looking for the 35 // <head>. Pages are read lazily and abandoned at </head>, so a link to a 36 // huge file costs at most this much, and usually nothing: non-HTML content 37 // types are rejected before the body is touched. 38 maxPreviewBytes = 4 << 20 39 maxPreviewImage = 2 << 20 40 previewTimeout = 10 * time.Second 41 maxRedirects = 3 42) 43 44// Preview kinds. Only "card" is rendered; the rest are recorded so that we do 45// not fetch the same link again. 46const ( 47 kindCard = "card" 48 kindImage = "image" // the link is an image 49 kindOther = "other" // not HTML (a download, a PDF, ...) or unreadable 50 kindNone = "none" // HTML, but without Open Graph data 51) 52 53type Preview struct { 54 URL string `json:"url"` 55 Kind string `json:"kind"` 56 Title string `json:"title,omitempty"` 57 Description string `json:"description,omitempty"` 58 Image string `json:"image,omitempty"` 59 Site string `json:"site,omitempty"` 60} 61 62var ( 63 imageMarkdown = regexp.MustCompile(`!\[[^\]]*\]\([^)]*\)`) 64 linkPattern = regexp.MustCompile(`https?://[^\s<>]+`) 65) 66 67// firstLink returns the first http(s) URL in a message body, or "". Markdown 68// image embeds (uploads, gifs) are skipped: they are already shown inline. 69func firstLink(body string) string { 70 loc := linkPattern.FindString(imageMarkdown.ReplaceAllString(body, " ")) 71 // Trailing punctuation belongs to the sentence, not the URL. A closing 72 // paren is kept only when the URL has a matching opening one. 73 for loc != "" { 74 last := loc[len(loc)-1] 75 switch { 76 case strings.IndexByte(".,;:!?'\"]}*_", last) >= 0: 77 case last == ')' && strings.Count(loc, ")") > strings.Count(loc, "("): 78 default: 79 u, err := url.Parse(loc) 80 if err != nil || u.Host == "" { 81 return "" 82 } 83 return loc 84 } 85 loc = loc[:len(loc)-1] 86 } 87 return "" 88} 89 90var errBlockedAddr = errors.New("address not allowed") 91 92// Ranges that are not covered by the net.IP predicates but are not the public 93// internet either. 94var extraBlocked = func() []*net.IPNet { 95 var out []*net.IPNet 96 for _, c := range []string{"100.64.0.0/10", "198.18.0.0/15", "192.0.0.0/24", "240.0.0.0/4", "0.0.0.0/8"} { 97 _, n, _ := net.ParseCIDR(c) 98 out = append(out, n) 99 } 100 return out 101}() 102 103// publicIP reports whether ip is somewhere a user-supplied link may send us. 104func publicIP(ip net.IP) bool { 105 if ip == nil || ip.IsLoopback() || ip.IsPrivate() || ip.IsUnspecified() || 106 ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsMulticast() { 107 return false 108 } 109 for _, n := range extraBlocked { 110 if n.Contains(ip) { 111 return false 112 } 113 } 114 return true 115} 116 117type previewer struct { 118 client *http.Client 119 mediaDir string 120 sem chan struct{} 121} 122 123// newPreviewer builds the fetcher. The address check runs inside the dialer, 124// on the IP actually being connected to, so it covers redirects and DNS tricks 125// (rebinding) that checking the URL up front would miss. allowPrivate exists 126// for tests that talk to a local server. 127func newPreviewer(mediaDir string, allowPrivate bool) *previewer { 128 dialer := &net.Dialer{ 129 Timeout: 5 * time.Second, 130 Control: func(_, address string, _ syscall.RawConn) error { 131 host, _, err := net.SplitHostPort(address) 132 if err != nil || (!allowPrivate && !publicIP(net.ParseIP(host))) { 133 return errBlockedAddr 134 } 135 return nil 136 }, 137 } 138 return &previewer{ 139 mediaDir: mediaDir, 140 sem: make(chan struct{}, 4), 141 client: &http.Client{ 142 Timeout: previewTimeout, 143 Transport: &http.Transport{ 144 Proxy: nil, // a proxy would bypass the dial check 145 DialContext: dialer.DialContext, 146 TLSHandshakeTimeout: 5 * time.Second, 147 ResponseHeaderTimeout: 5 * time.Second, 148 MaxResponseHeaderBytes: 64 << 10, 149 DisableKeepAlives: true, 150 }, 151 CheckRedirect: func(req *http.Request, via []*http.Request) error { 152 if len(via) > maxRedirects { 153 return errors.New("too many redirects") 154 } 155 if req.URL.Scheme != "http" && req.URL.Scheme != "https" { 156 return errors.New("bad redirect scheme") 157 } 158 return nil 159 }, 160 }, 161 } 162} 163 164func (p *previewer) get(ctx context.Context, rawURL, accept string) (*http.Response, error) { 165 req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) 166 if err != nil { 167 return nil, err 168 } 169 req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; irvc-link-preview)") 170 req.Header.Set("Accept", accept) 171 req.Header.Set("Accept-Language", "en") 172 resp, err := p.client.Do(req) 173 if err != nil { 174 return nil, err 175 } 176 if resp.StatusCode < 200 || resp.StatusCode > 299 { 177 resp.Body.Close() 178 return nil, errors.New("status " + resp.Status) 179 } 180 return resp, nil 181} 182 183// fetch builds the preview for link. It returns nil only when the link could 184// not be fetched at all, so a transient failure is retried on the next edit. 185func (p *previewer) fetch(ctx context.Context, link string) *Preview { 186 resp, err := p.get(ctx, link, "text/html,application/xhtml+xml;q=0.9,*/*;q=0.1") 187 if err != nil { 188 return nil 189 } 190 defer resp.Body.Close() 191 out := &Preview{URL: link, Kind: kindOther} 192 193 ctype := resp.Header.Get("Content-Type") 194 media, _, _ := mime.ParseMediaType(ctype) 195 switch { 196 case strings.HasPrefix(media, "image/"): 197 out.Kind = kindImage 198 return out 199 case media != "text/html" && media != "application/xhtml+xml": 200 return out 201 } 202 203 og := parseOpenGraph(io.LimitReader(resp.Body, maxPreviewBytes), ctype) 204 if og["title"] == "" && og["description"] == "" && og["image"] == "" { 205 out.Kind = kindNone 206 return out 207 } 208 out.Kind = kindCard 209 out.Title = clip(og["title"], 200) 210 out.Description = clip(og["description"], 300) 211 out.Site = clip(og["site_name"], 80) 212 if img := og["image"]; img != "" && resp.Request != nil { 213 if ref, err := resp.Request.URL.Parse(img); err == nil && (ref.Scheme == "http" || ref.Scheme == "https") { 214 out.Image = p.cacheImage(ctx, ref.String()) 215 } 216 } 217 return out 218} 219 220// parseOpenGraph reads og:* meta tags from the document head, stopping at 221// </head> or <body> (or the end of r) rather than parsing the whole page. 222func parseOpenGraph(r io.Reader, contentType string) map[string]string { 223 og := map[string]string{} 224 if cr, err := charset.NewReader(r, contentType); err == nil { 225 r = cr 226 } 227 z := html.NewTokenizer(r) 228 for { 229 switch z.Next() { 230 case html.ErrorToken: 231 return og 232 case html.EndTagToken: 233 if name, _ := z.TagName(); string(name) == "head" { 234 return og 235 } 236 case html.StartTagToken, html.SelfClosingTagToken: 237 name, hasAttr := z.TagName() 238 switch string(name) { 239 case "body": 240 return og 241 case "meta": 242 if !hasAttr { 243 continue 244 } 245 var prop, content string 246 for { 247 k, v, more := z.TagAttr() 248 switch string(k) { 249 case "property", "name": 250 if prop == "" { 251 prop = strings.ToLower(string(v)) 252 } 253 case "content": 254 content = string(v) 255 } 256 if !more { 257 break 258 } 259 } 260 if key, ok := strings.CutPrefix(prop, "og:"); ok && og[key] == "" { 261 og[key] = cleanText(content) 262 } 263 } 264 } 265 } 266} 267 268var entity = regexp.MustCompile(`&(#[0-9]+|#[xX][0-9a-fA-F]+|[A-Za-z][A-Za-z0-9]*);`) 269 270// cleanText turns a meta tag value into plain text for display. The tokenizer 271// has already decoded the attribute once (so & is & and ’ is ’), but 272// plenty of sites encode twice (&amp;), which would show up as a literal 273// "&" in the card. So: decode once more if entities remain, drop control 274// characters and collapse whitespace. The result is plain text, never HTML; 275// the client renders it as a text node, so a title containing "<b>" displays as 276// those characters rather than being interpreted. 277func cleanText(s string) string { 278 s = strings.ToValidUTF8(s, "") 279 if entity.MatchString(s) { 280 s = html.UnescapeString(s) 281 } 282 s = strings.Map(func(r rune) rune { 283 if unicode.IsControl(r) && !unicode.IsSpace(r) { 284 return -1 285 } 286 return r 287 }, s) 288 return strings.Join(strings.Fields(s), " ") 289} 290 291func clip(s string, n int) string { 292 if utf8.RuneCountInString(s) <= n { 293 return s 294 } 295 r := []rune(s) 296 return strings.TrimSpace(string(r[:n-1])) + "…" 297} 298 299// cacheImage downloads a preview thumbnail into the media store and returns 300// its local URL, or "" if it is not a small raster image. 301func (p *previewer) cacheImage(ctx context.Context, src string) string { 302 resp, err := p.get(ctx, src, "image/*") 303 if err != nil { 304 return "" 305 } 306 defer resp.Body.Close() 307 data, err := io.ReadAll(io.LimitReader(resp.Body, maxPreviewImage+1)) 308 if err != nil || len(data) > maxPreviewImage || len(data) == 0 { 309 return "" 310 } 311 ct := http.DetectContentType(data) 312 ext, ok := mediaExt[ct] 313 if !ok || !strings.HasPrefix(ct, "image/") { 314 return "" 315 } 316 name := randToken() + ext 317 if err := os.WriteFile(filepath.Join(p.mediaDir, name), data, 0o644); err != nil { 318 return "" 319 } 320 return "/api/media/" + name 321} 322 323func (s *Server) messagePreview(msgID int64) *Preview { 324 var raw sql.NullString 325 if s.db.QueryRow(`SELECT preview FROM messages WHERE id = ?`, msgID).Scan(&raw) != nil { 326 return nil 327 } 328 return decodePreview(raw.String) 329} 330 331func decodePreview(raw string) *Preview { 332 if raw == "" { 333 return nil 334 } 335 var p Preview 336 if json.Unmarshal([]byte(raw), &p) != nil { 337 return nil 338 } 339 return &p 340} 341 342// refreshPreview brings a message's stored preview in line with its current 343// body. It runs in the background after a message is sent or edited, and tells 344// clients (as an edit event) only when a card appears or goes away. 345func (s *Server) refreshPreview(msgID int64) { 346 s.previews.sem <- struct{}{} 347 defer func() { <-s.previews.sem }() 348 349 var body string 350 var raw sql.NullString 351 var channelID int64 352 if s.db.QueryRow(`SELECT body, preview, channel_id FROM messages WHERE id = ?`, msgID).Scan(&body, &raw, &channelID) != nil { 353 return 354 } 355 old := decodePreview(raw.String) 356 link := firstLink(body) 357 358 var next *Preview 359 if link != "" { 360 if old != nil && old.URL == link { 361 return 362 } 363 ctx, cancel := context.WithTimeout(context.Background(), previewTimeout) 364 defer cancel() 365 if next = s.previews.fetch(ctx, link); next == nil { 366 return 367 } 368 } 369 370 stored := "" 371 if next != nil { 372 b, _ := json.Marshal(next) 373 stored = string(b) 374 } 375 // Only write if the preview is still what we started from: a removal (or 376 // another refresh) that landed while we were fetching wins. 377 res, err := s.db.Exec( 378 `UPDATE messages SET preview = NULLIF(?, '') WHERE id = ? AND COALESCE(preview, '') = ?`, 379 stored, msgID, raw.String, 380 ) 381 if err != nil { 382 return 383 } 384 if n, _ := res.RowsAffected(); n == 0 { 385 return 386 } 387 if (old != nil && old.Kind == kindCard) || (next != nil && next.Kind == kindCard) { 388 s.broadcastMessage(msgID, channelID) 389 } 390} 391 392// broadcastMessage re-reads a message and sends it to the channel's audience as 393// an edit, so clients replace their copy. 394func (s *Server) broadcastMessage(msgID, channelID int64) { 395 if m, chType, chName, ok := s.loadMessage(msgID, channelID); ok { 396 s.notifyChannel(chType, chName, event{Type: "message_edit", Message: &m}) 397 } 398} 399 400// loadMessage reads a message as clients see it, plus its channel's type and 401// name for routing. 402func (s *Server) loadMessage(msgID, channelID int64) (m Message, chType, chName string, ok bool) { 403 var raw sql.NullString 404 err := s.db.QueryRow( 405 `SELECT m.id, m.channel_id, m.user_id, u.username, m.body, m.created_at, COALESCE(m.edited_at, 0), m.preview 406 FROM messages m JOIN users u ON u.id = m.user_id WHERE m.id = ?`, msgID, 407 ).Scan(&m.ID, &m.ChannelID, &m.UserID, &m.Username, &m.Body, &m.CreatedAt, &m.EditedAt, &raw) 408 if err != nil { 409 return m, "", "", false 410 } 411 m.Preview = decodePreview(raw.String) 412 m.Reactions = s.messageReactions(m.ID) 413 s.db.QueryRow(`SELECT type, name FROM channels WHERE id = ?`, channelID).Scan(&chType, &chName) 414 return m, chType, chName, true 415} 416 417var errNoPreview = errors.New("no preview") 418 419// removePreview deletes the card from a message its author owns. The link is 420// remembered as "none" so the same link is not fetched again on a later edit, 421// and the cached thumbnail is deleted along with it. It returns the message's 422// channel for notification. 423func (s *Server) removePreview(msgID, userID int64) (channelID int64, err error) { 424 var raw sql.NullString 425 if err := s.db.QueryRow( 426 `SELECT channel_id, preview FROM messages WHERE id = ? AND user_id = ?`, msgID, userID, 427 ).Scan(&channelID, &raw); err != nil { 428 return 0, err 429 } 430 old := decodePreview(raw.String) 431 if old == nil || old.Kind != kindCard { 432 return channelID, errNoPreview 433 } 434 b, _ := json.Marshal(Preview{URL: old.URL, Kind: kindNone}) 435 if _, err := s.db.Exec(`UPDATE messages SET preview = ? WHERE id = ?`, string(b), msgID); err != nil { 436 return 0, err 437 } 438 if name, ok := strings.CutPrefix(old.Image, "/api/media/"); ok && mediaName.MatchString(name) { 439 os.Remove(filepath.Join(s.cfg.MediaDir, name)) 440 } 441 return channelID, nil 442} 443 444func (s *Server) handleDeletePreview(c *gin.Context) { 445 msgID, err := strconv.ParseInt(c.Param("id"), 10, 64) 446 if err != nil { 447 c.JSON(http.StatusBadRequest, gin.H{"error": "bad message"}) 448 return 449 } 450 user := c.MustGet("user").(User) 451 channelID, err := s.removePreview(msgID, user.ID) 452 switch { 453 case errors.Is(err, sql.ErrNoRows): 454 c.JSON(http.StatusNotFound, gin.H{"error": "message not found"}) 455 return 456 case errors.Is(err, errNoPreview): 457 // Already gone: nothing to do, and the caller's goal is met. 458 case err != nil: 459 c.JSON(http.StatusInternalServerError, gin.H{"error": "db"}) 460 return 461 } 462 m, chType, chName, ok := s.loadMessage(msgID, channelID) 463 if !ok { 464 c.JSON(http.StatusInternalServerError, gin.H{"error": "db"}) 465 return 466 } 467 s.notifyChannel(chType, chName, event{Type: "message_edit", Message: &m}) 468 c.JSON(http.StatusOK, m) 469}