Just a sentence.
package extract import ( "errors" "net/url" "os" "strings" "testing" "time" "golang.org/x/net/html" ) func extractFile(t *testing.T, name, pageURL string) (*Article, error) { t.Helper() data, err := os.ReadFile("../../testdata/pages/" + name) if err != nil { t.Fatal(err) } u, err := url.Parse(pageURL) if err != nil { t.Fatal(err) } return Extract(parse(t, string(data)), u) } func parse(t *testing.T, page string) *html.Node { t.Helper() doc, err := html.Parse(strings.NewReader(page)) if err != nil { t.Fatal(err) } return doc } func render(t *testing.T, n *html.Node) string { t.Helper() var b strings.Builder for c := n.FirstChild; c != nil; c = c.NextSibling { if err := html.Render(&b, c); err != nil { t.Fatal(err) } } return b.String() } func TestExtractBlog(t *testing.T) { a, err := extractFile(t, "blog.html", "https://reading.example.com/posts/eink") if err != nil { t.Fatal(err) } checks := []struct{ field, got, want string }{ {"Title", a.Title, "Why E-Ink Is Easier on the Eyes"}, {"Author", a.Author, "Ada Lovelace"}, {"SiteName", a.SiteName, "The Reading Desk"}, {"Language", a.Language, "en"}, {"URL", a.URL, "https://reading.example.com/posts/eink"}, } for _, c := range checks { if c.got != c.want { t.Errorf("%s = %q, want %q", c.field, c.got, c.want) } } if !strings.HasPrefix(a.Description, "A look at how electronic paper") { t.Errorf("Description = %q", a.Description) } if want := time.Date(2025, 3, 14, 0, 0, 0, 0, time.UTC); !a.Published.Equal(want) { t.Errorf("Published = %v, want %v", a.Published, want) } content := render(t, a.Content) for _, want := range []string{ "Electronic paper does not shine light into your eyes.", "
", "Readable in direct sunlight ", "hard to beat & well worth", // Links are made absolute. ``, } { if !strings.Contains(content, want) { t.Errorf("content is missing %q", want) } } for _, unwanted := range []string{ "Subscribe to our newsletter", // navigation "We use cookies", // cookie banner "Related posts", // sidebar "Great article, thanks!", // comments "All rights reserved", // footer "window.dataLayer", // script "", // title repeated in the content } { if strings.Contains(content, unwanted) { t.Errorf("content contains %q", unwanted) } } } func TestExtractNoContent(t *testing.T) { pages := map[string]string{ "JavaScript shell": `
App `, "too short": `Short `, "empty": ``, } u, _ := url.Parse("https://example.com/") for name, page := range pages { if _, err := Extract(parse(t, page), u); !errors.Is(err, ErrNoContent) { t.Errorf("%s: error = %v, want ErrNoContent", name, err) } } } func TestLanguageTag(t *testing.T) { tests := map[string]string{ "en": "en", "en-US": "en-US", "en_GB": "en-GB", " fr ": "fr", "pt-br": "pt-BR", "": "en", "not a language": "en", "und": "en", } for in, want := range tests { if got := languageTag(in); got != want { t.Errorf("languageTag(%q) = %q, want %q", in, got, want) } } } func TestRemoveLeadingTitle(t *testing.T) { tests := []struct { name, body, title, want string }{ {"matching h1", " Just a sentence.
The Title
text
", "The Title", "text
"}, {"case and spacing", "\nthe TITLE
text
", "The Title", "text
"}, {"matching h2", "The Title
text
", "The Title", "text
"}, {"different heading", "Other
text
", "The Title", "Other
text
"}, {"not first", "intro
The Title
", "The Title", "intro
The Title
"}, {"paragraph", "The Title
", "The Title", "The Title
"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { doc, err := html.Parse(strings.NewReader("" + tt.body + "")) if err != nil { t.Fatal(err) } body := doc.FirstChild.LastChild // html > body removeLeadingTitle(body, tt.title) got := strings.TrimSpace(render(t, body)) if got != tt.want { t.Errorf("got %q, want %q", got, tt.want) } }) } }