package clean import ( "net/url" "strings" "testing" "golang.org/x/net/html" ) var base, _ = url.Parse("https://example.com/posts/article") // body parses markup as the children of a , like extracted content. func body(t *testing.T, markup string) *html.Node { t.Helper() doc, err := html.Parse(strings.NewReader("" + markup + "")) if err != nil { t.Fatal(err) } return doc.FirstChild.LastChild // html > body } func cleanString(t *testing.T, markup string) string { t.Helper() cleaned, err := Content(body(t, markup), base) if err != nil { t.Fatalf("Content(%q): %v", markup, err) } out, err := Render(cleaned) if err != nil { t.Fatalf("Render(%q): %v", markup, err) } if err := WellFormed("
" + out + "
"); err != nil { t.Fatalf("Content(%q) = %q, not well-formed: %v", markup, out, err) } return out } func TestContentSanitizes(t *testing.T) { tests := []struct { name, in, want string }{ {"keeps basic markup", `

Hello there you.

`, `

Hello there you.

`}, {"drops scripts and their content", `

a

b

`, `

a

b

`}, {"drops styles and their content", `

a

`, `

a

`}, {"drops event handlers, classes, ids and inline styles", `

a

`, `

a

`}, {"drops forms but keeps their text", `

Sign up

`, `

Sign up

Go`}, {"drops iframes and objects", `

a

`, `

a

`}, {"drops javascript links but keeps the text", `

click

`, `

click

`}, {"makes relative links absolute", `

about other notes

`, `

about other notes

`}, {"keeps mailto links", `

mail

`, `

mail

`}, {"keeps valid table spans and drops invalid ones", `
a
`, `
a
`}, {"keeps valid lang and dir, drops invalid", `

a

b

`, `

a

b

`}, {"keeps images in paragraphs and makes their src absolute", `

a

x

`, `

a

x

`}, {"drops images with unsafe sources", `

a x

`, `

a

`}, {"removes empty paragraphs", `

a


b

`, `

a

b

`}, {"repairs mailto and tel links mangled by go-trafilatura", `

mail call

`, `

mail call

`}, {"restores code blocks from go-trafilatura's block code", "a = 1\nb = 2\n

after

", "
a = 1\nb = 2\n

after

"}, {"restores multi-line code inside a list item", "", ""}, {"leaves inline code in a paragraph alone, even with a line break", "

Run make\nbuild now.

", "

Run make\nbuild now.

"}, {"keeps code blocks", `
if a < b {
	return
}
`, "
if a < b {\n\treturn\n}
"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := cleanString(t, tt.in); got != tt.want { t.Errorf("got %q\nwant %q", got, tt.want) } }) } } func TestContentRepairsNesting(t *testing.T) { // Each of these is invalid HTML that must still come out well-formed. for _, in := range []string{ `

a

b
c

`, ``, `outer inner`, `
no row
`, `

unclosed emphasis

next`, `

  • orphan item
    • `, } { cleanString(t, in) // fails the test if not well-formed } } func TestContentEntities(t *testing.T) { //   and other HTML entities aren't defined in XML; they must come out // as characters. got := cleanString(t, `

    a b © — <tag> &

    `) want := "

    a b © — <tag> &

    " if got != want { t.Errorf("got %q, want %q", got, want) } } func TestShiftHeadings(t *testing.T) { tests := []struct{ in, want string }{ {`

    a

    b

    c

    `, `

    a

    b

    c

    `}, {`

    a

    b

    `, `

    a

    b

    `}, {`

    a

    b

    `, `

    a

    b

    `}, {`

    a

    b
    c
    `, `

    a

    b
    c
    `}, {`

    no headings

    `, `

    no headings

    `}, } for _, tt := range tests { if got := cleanString(t, tt.in); got != tt.want { t.Errorf("%s: got %q, want %q", tt.in, got, tt.want) } } } func TestWellFormed(t *testing.T) { good := []string{`

    a


    `, `

    & < > " '  

    `} bad := []string{`

    unclosed`, `

    a`, `

     

    `, `
    `} for _, s := range good { if err := WellFormed(s); err != nil { t.Errorf("WellFormed(%q) = %v, want nil", s, err) } } for _, s := range bad { if err := WellFormed(s); err == nil { t.Errorf("WellFormed(%q) = nil, want an error", s) } } }