linkinfo/default.go

180 lines
3.2 KiB
Go
Raw Normal View History

package linkinfo
2019-10-03 23:44:38 +00:00
import (
"errors"
"fmt"
"github.com/PuerkitoBio/goquery"
"io"
"io/ioutil"
"net/http"
"net/url"
"path"
"strconv"
"strings"
)
const (
contentTypeHtml = "text/html"
maxBodySizeBytes = 20971520
2019-10-03 23:44:38 +00:00
)
func (api *LinkInfoApi) DefaultLinkHandler(link string) (*LinkInfo, error) {
2019-10-03 23:44:38 +00:00
redirects := make([]string, 0)
u, err := url.Parse(link)
if err != nil {
return nil, err
}
var res *http.Response
for i := 0; i < 10; i++ {
res, err = api.Client.Head(link)
2019-10-03 23:44:38 +00:00
if err != nil {
return nil, err
}
if (res.StatusCode == 301 || res.StatusCode == 302) && res.Header.Get("Location") != "" {
link = res.Header.Get("Location")
redirects = append(redirects, link)
} else {
break
}
}
if res != nil && res.StatusCode != 200 {
return nil, errors.New("invalid response, expected 200, got " + strconv.Itoa(res.StatusCode))
}
contentType := res.Header.Get("Content-Type")
if contentType == "" {
contentType = api.detectContentType(link, "application/octet-stream")
2019-10-03 23:44:38 +00:00
}
if idx := strings.Index(contentType, ";"); idx != -1 {
contentType = contentType[:idx]
}
var contentLength int64
if contentLengthStr := res.Header.Get("Content-Length"); contentLengthStr != "" {
contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64)
}
ret := &LinkInfo{
ContentType: contentType,
2019-10-03 23:44:38 +00:00
ContentLength: contentLength,
}
switch contentType {
case contentTypeHtml:
if contentLength >= 0 && contentLength < maxBodySizeBytes {
err = api.retrieveHtmlLinkTitle(ret, link)
break
}
fallthrough
2019-10-03 23:44:38 +00:00
default:
ret.Title = fmt.Sprintf("%s (%s, %s)", path.Base(u.Path), contentType, ByteCountDecimal(contentLength))
}
return ret, err
}
func (api *LinkInfoApi) detectContentType(link, defaultType string) string {
2019-10-03 23:44:38 +00:00
req, err := http.NewRequest("GET", link, nil)
if err != nil {
return defaultType
}
req.Header.Set("Range", "bytes=0-512")
res, err := api.Client.Do(req)
2019-10-03 23:44:38 +00:00
if err != nil {
return defaultType
}
defer res.Body.Close()
b, err := ioutil.ReadAll(io.LimitReader(res.Body, 512))
if err != nil {
return defaultType
}
t := http.DetectContentType(b)
if t == "" {
t = defaultType
}
return t
}
var (
attrKeys = []string{"property", "name", "itemprop"}
)
func (api *LinkInfoApi) retrieveHtmlLinkTitle(i *LinkInfo, link string) error {
res, err := api.Client.Get(link)
2019-10-03 23:44:38 +00:00
if err != nil {
return err
}
defer res.Body.Close()
q, err := goquery.NewDocumentFromReader(io.LimitReader(res.Body, maxBodySizeBytes))
2019-10-03 23:44:38 +00:00
if err != nil {
return err
}
meta := q.Find("meta")
metaTags := make(map[string]string)
meta.Each(func(_ int, s *goquery.Selection) {
var key string
var exists bool
for _, k := range attrKeys {
key, exists = s.Attr(k)
if exists {
break
}
}
if key == "" {
return
}
})
var attr string
var exists bool
if attr, exists = metaTags["og:title"]; exists {
i.Title = attr
} else if tag := q.Find("title"); tag.Length() > 0 {
i.Title = strings.TrimSpace(tag.Text())
2019-10-03 23:44:38 +00:00
}
if attr, exists = metaTags["og:description"]; exists {
i.Description = attr
} else if attr, exists = metaTags["description"]; exists {
i.Description = attr
}
if attr, exists = metaTags["duration"]; exists {
i.Duration = attr
}
return nil
}