package extract import ( "encoding/json" "net/url" "strings" "time" "golang.org/x/net/html" "golang.org/x/net/html/atom" ) // pageMeta is metadata the page declares about itself, used to fill in // anything go-trafilatura didn't find, and for the cover image, which it // doesn't look for reliably. type pageMeta struct { Title, Author, SiteName, Description, Image string Published time.Time } // articleTypes are the schema.org types whose JSON-LD describes an article. var articleTypes = map[string]bool{ "Article": true, "NewsArticle": true, "BlogPosting": true, "TechArticle": true, "ReportageNewsArticle": true, "AnalysisNewsArticle": true, "OpinionNewsArticle": true, "ScholarlyArticle": true, "Report": true, } // readPageMeta collects Open Graph, article: and Twitter meta tags and // schema.org JSON-LD from doc. JSON-LD wins over meta tags, and meta tags // over . Relative image URLs are resolved against base. func readPageMeta(doc *html.Node, base *url.URL) pageMeta { tags := map[string]string{} // first value of each property/name, lowercased key var title string var ld []map[string]any walk(doc, func(n *html.Node) { switch n.DataAtom { case atom.Meta: key := strings.ToLower(attr(n, "property")) if key == "" { key = strings.ToLower(attr(n, "name")) } if v := strings.TrimSpace(attr(n, "content")); key != "" && v != "" { if _, seen := tags[key]; !seen { tags[key] = v } } case atom.Title: if title == "" { title = strings.TrimSpace(textContent(n)) } case atom.Script: if strings.Contains(strings.ToLower(attr(n, "type")), "ld+json") { ld = append(ld, articleObjects(textContent(n))...) } } }) m := pageMeta{ Title: first(tags["og:title"], tags["twitter:title"], title), Author: first(tags["article:author"], tags["author"], tags["twitter:creator"]), SiteName: first(tags["og:site_name"], tags["application-name"]), Description: first(tags["og:description"], tags["description"], tags["twitter:description"]), Image: first(tags["og:image"], tags["og:image:url"], tags["twitter:image"], tags["twitter:image:src"]), Published: parseDate(first(tags["article:published_time"], tags["date"], tags["dc.date"])), } // An article:author that is a profile URL isn't a name. if strings.HasPrefix(m.Author, "http") { m.Author = "" } for _, o := range ld { if s := str(o["headline"]); s != "" { m.Title = s } if s := names(o["author"]); s != "" { m.Author = s } if s := names(o["publisher"]); s != "" && m.SiteName == "" { m.SiteName = s } if s := str(o["description"]); s != "" && m.Description == "" { m.Description = s } if s := imageURL(o["image"]); s != "" && m.Image == "" { m.Image = s } if t := parseDate(str(o["datePublished"])); !t.IsZero() { m.Published = t } } if m.Image != "" { if u, err := base.Parse(m.Image); err == nil && (u.Scheme == "http" || u.Scheme == "https") { m.Image = u.String() } else { m.Image = "" } } return m } // articleObjects returns the article objects in a JSON-LD script, looking // inside arrays and @graph. Invalid JSON yields nothing. func articleObjects(script string) []map[string]any { var v any if json.Unmarshal([]byte(script), &v) != nil { return nil } var out []map[string]any var visit func(any) visit = func(v any) { switch v := v.(type) { case []any: for _, e := range v { visit(e) } case map[string]any: if isArticle(v["@type"]) { out = append(out, v) } visit(v["@graph"]) } } visit(v) return out } func isArticle(t any) bool { switch t := t.(type) { case string: return articleTypes[t] case []any: for _, e := range t { if s, ok := e.(string); ok && articleTypes[s] { return true } } } return false } // names reads an author or publisher: a string, an object with a name, or // a list of those, joined with ", ". func names(v any) string { switch v := v.(type) { case string: return strings.TrimSpace(v) case map[string]any: return str(v["name"]) case []any: var out []string for _, e := range v { if s := names(e); s != "" { out = append(out, s) } } return strings.Join(out, ", ") } return "" } // imageURL reads a schema.org image: a URL, an ImageObject, or a list. func imageURL(v any) string { switch v := v.(type) { case string: return strings.TrimSpace(v) case map[string]any: return first(str(v["url"]), str(v["contentUrl"])) case []any: for _, e := range v { if s := imageURL(e); s != "" { return s } } } return "" } func str(v any) string { s, _ := v.(string) return strings.TrimSpace(s) } func first(values ...string) string { for _, v := range values { if v = strings.TrimSpace(v); v != "" { return v } } return "" } // parseDate accepts the ISO 8601 forms pages use for publication dates. func parseDate(s string) time.Time { for _, layout := range []string{time.RFC3339, "2006-01-02T15:04:05Z0700", "2006-01-02T15:04:05", "2006-01-02T15:04", "2006-01-02"} { if t, err := time.Parse(layout, strings.TrimSpace(s)); err == nil { return t } } return time.Time{} } func attr(n *html.Node, key string) string { for _, a := range n.Attr { if a.Key == key { return a.Val } } return "" } func walk(n *html.Node, f func(*html.Node)) { if n.Type == html.ElementNode { f(n) } for c := n.FirstChild; c != nil; c = c.NextSibling { walk(c, f) } }