ReKoPub/internal/web/server_test.go

package web

import (
	"context"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"slices"
	"strings"
	"sync"
	"testing"
	"time"
)

// fakeCLI stands in for the rekopub command. It writes a book named after
// the page's last path segment into the -SAVE folder, as the command does.
type fakeCLI struct {
	mu    sync.Mutex
	calls [][]string
	fail  runResult     // returned instead of a book when Code != 0
	block chan struct{} // if set, runs wait for it to close
}

func (f *fakeCLI) run(ctx context.Context, args []string) runResult {
	f.mu.Lock()
	f.calls = append(f.calls, args)
	f.mu.Unlock()
	if f.block != nil {
		select {
		case <-f.block:
		case <-ctx.Done():
			return runResult{Code: -1}
		}
	}
	if f.fail.Code != 0 {
		return f.fail
	}
	var page, dir string
	kepub := false
	for _, a := range args {
		switch {
		case strings.HasPrefix(a, "-URL="):
			u, _ := url.Parse(strings.TrimPrefix(a, "-URL="))
			page = filepath.Base(u.Path)
		case strings.HasPrefix(a, "-SAVE="):
			dir = strings.TrimPrefix(a, "-SAVE=")
		case a == "-KEPUB":
			kepub = true
		}
	}
	path := filepath.Join(dir, page+".epub")
	if kepub {
		path = filepath.Join(dir, page+".kepub.epub")
	}
	if err := os.WriteFile(path, []byte("book:"+page), 0o640); err != nil {
		return runResult{Code: 4, Err: err.Error()}
	}
	abs, _ := filepath.Abs(path) // the command prints absolute paths
	return runResult{Path: abs}
}

func (f *fakeCLI) count() int {
	f.mu.Lock()
	defer f.mu.Unlock()
	return len(f.calls)
}

type testServer struct {
	*Server
	cli  *fakeCLI
	now  time.Time
	http http.Handler
}

func newTestServer(t *testing.T, cfg Config) *testServer {
	t.Helper()
	cfg.CLI = "rekopub"
	// Relative, as on the server (DATA_DIR=data).
	t.Chdir(t.TempDir())
	cfg.Dir = filepath.Join("data", "books")
	s, err := New(cfg)
	if err != nil {
		t.Fatal(err)
	}
	ts := &testServer{Server: s, cli: &fakeCLI{}, now: time.Now()}
	s.run = ts.cli.run
	s.now = func() time.Time { return ts.now }
	s.isLocal = func(_ context.Context, host string) bool { return host == "localhost" || host == "10.0.0.1" }
	ts.http = s.Handler()
	return ts
}

type response struct {
	code int
	body string
	rep  reply
	hdr  http.Header
}

func (ts *testServer) do(t *testing.T, req *http.Request) response {
	t.Helper()
	w := httptest.NewRecorder()
	ts.http.ServeHTTP(w, req)
	res := response{code: w.Code, body: w.Body.String(), hdr: w.Header()}
	if strings.HasPrefix(w.Header().Get("Content-Type"), "application/json") {
		if err := json.Unmarshal(w.Body.Bytes(), &res.rep); err != nil {
			t.Fatalf("bad JSON %q: %v", res.body, err)
		}
	}
	return res
}

// post submits the form as the page's script does, from visitor ip.
func (ts *testServer) post(t *testing.T, ip, page, format string) response {
	t.Helper()
	form := url.Values{"url": {page}}
	if format != "" {
		form.Set("format", format)
	}
	req := httptest.NewRequest("POST", "/api/convert", strings.NewReader(form.Encode()))
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	req.Header.Set("Accept", "application/json")
	req.Header.Set("X-Real-IP", ip)
	return ts.do(t, req)
}

func (ts *testServer) get(t *testing.T, ip, path string) response {
	t.Helper()
	req := httptest.NewRequest("GET", path, nil)
	req.Header.Set("X-Real-IP", ip)
	return ts.do(t, req)
}

func TestConvertEPUB(t *testing.T) {
	ts := newTestServer(t, Config{})
	res := ts.post(t, "203.0.113.5", "https://example.com/posts/hello#top", "")
	if res.code != http.StatusOK || len(res.rep.Files) != 1 {
		t.Fatalf("code = %d, body = %s", res.code, res.body)
	}
	if res.rep.KeepHours != 24 {
		t.Errorf("keepHours = %d, want 24", res.rep.KeepHours)
	}
	f := res.rep.Files[0]
	if f.Format != "EPUB" || f.Name != "hello.epub" || f.Size != int64(len("book:hello")) {
		t.Errorf("file = %+v", f)
	}
	args := ts.cli.calls[0]
	if len(args) != 2 || args[0] != "-URL=https://example.com/posts/hello" ||
		!strings.HasPrefix(args[1], "-SAVE="+ts.cfg.Dir+string(filepath.Separator)) || !strings.HasSuffix(args[1], string(filepath.Separator)) {
		t.Errorf("command args = %q", args)
	}

	dl := ts.get(t, "198.51.100.7", f.URL)
	if dl.code != http.StatusOK || dl.body != "book:hello" {
		t.Fatalf("download: code = %d, body = %q", dl.code, dl.body)
	}
	if got := dl.hdr.Get("Content-Disposition"); got != `attachment; filename="hello.epub"` {
		t.Errorf("Content-Disposition = %q", got)
	}
	if got := dl.hdr.Get("Content-Type"); got != "application/epub+zip" {
		t.Errorf("Content-Type = %q", got)
	}
	if got := dl.hdr.Get("X-Content-Type-Options"); got != "nosniff" {
		t.Errorf("X-Content-Type-Options = %q", got)
	}
}

func TestConvertBoth(t *testing.T) {
	ts := newTestServer(t, Config{})
	res := ts.post(t, "203.0.113.5", "https://example.com/a", "both")
	if res.code != http.StatusOK || len(res.rep.Files) != 2 {
		t.Fatalf("code = %d, body = %s", res.code, res.body)
	}
	if res.rep.Files[0].Name != "a.epub" || res.rep.Files[1].Name != "a.kepub.epub" || res.rep.Files[1].Format != "KEPUB" {
		t.Errorf("files = %+v", res.rep.Files)
	}
	if path.Dir(res.rep.Files[0].URL) != path.Dir(res.rep.Files[1].URL) {
		t.Errorf("books of one conversion are in different folders: %+v", res.rep.Files)
	}
	if !slices.Contains(ts.cli.calls[1], "-KEPUB") {
		t.Errorf("second run args = %q, want -KEPUB", ts.cli.calls[1])
	}
	// Both books count as one conversion.
	ts.Server.mu.Lock()
	n := len(ts.all)
	ts.Server.mu.Unlock()
	if n != 1 {
		t.Errorf("counted %d conversions, want 1", n)
	}
}

func TestConvertKEPUB(t *testing.T) {
	ts := newTestServer(t, Config{})
	res := ts.post(t, "203.0.113.5", "https://example.com/a", "kepub")
	if res.code != http.StatusOK || len(res.rep.Files) != 1 || res.rep.Files[0].Name != "a.kepub.epub" {
		t.Fatalf("code = %d, body = %s", res.code, res.body)
	}
}

func TestBadRequests(t *testing.T) {
	tests := []struct {
		name, page, format, want string
	}{
		{"empty", "", "", "Enter the address"},
		{"no scheme", "example.com/a", "", "starting with http"},
		{"ftp", "ftp://example.com/a", "", "starting with http"},
		{"javascript", "javascript:alert(1)", "", "starting with http"},
		{"file", "file:///etc/passwd", "", "starting with http"},
		{"no host", "https:///a", "", "starting with http"},
		{"credentials", "https://user:pw@example.com/a", "", "user name or password"},
		{"port", "https://example.com:3000/a", "", "standard web ports"},
		{"control", "https://example.com/a\nb", "", "valid web address"},
		{"too long", "https://example.com/" + strings.Repeat("a", maxURLLen), "", "too long"},
		{"localhost", "http://localhost/a", "", "local or private"},
		{"private", "http://10.0.0.1/a", "", "local or private"},
		{"format", "https://example.com/a", "pdf", "Choose EPUB"},
	}
	ts := newTestServer(t, Config{Limit: 3})
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			res := ts.post(t, "203.0.113.5", tt.page, tt.format)
			if res.code != http.StatusBadRequest || !strings.Contains(res.rep.Error, tt.want) {
				t.Errorf("code = %d, error = %q; want 400 and %q", res.code, res.rep.Error, tt.want)
			}
		})
	}
	if n := ts.cli.count(); n != 0 {
		t.Errorf("command ran %d times, want 0", n)
	}
	// Rejected requests don't use up conversions.
	if res := ts.get(t, "203.0.113.5", "/api/status"); res.rep.Paused {
		t.Errorf("paused after rejected requests: %+v", res.rep)
	}
}

func TestOversizedForm(t *testing.T) {
	ts := newTestServer(t, Config{})
	res := ts.post(t, "203.0.113.5", "https://example.com/"+strings.Repeat("a", maxBodyLen), "")
	if res.code != http.StatusBadRequest {
		t.Errorf("code = %d, want 400", res.code)
	}
}

func TestLimitForEveryone(t *testing.T) {
	ts := newTestServer(t, Config{Limit: 3, PerVisitor: 10, Window: 15 * time.Minute})
	for i, ip := range []string{"203.0.113.1", "203.0.113.2", "203.0.113.3"} {
		if res := ts.post(t, ip, "https://example.com/a", ""); res.code != http.StatusOK {
			t.Fatalf("conversion %d: code = %d, body = %s", i+1, res.code, res.body)
		}
		ts.now = ts.now.Add(time.Minute)
	}
	res := ts.post(t, "203.0.113.4", "https://example.com/a", "")
	want := "Conversions are temporarily paused. Please try again in 12 minutes."
	if res.code != http.StatusTooManyRequests || !res.rep.Paused || res.rep.Error != want {
		t.Errorf("code = %d, reply = %+v; want 429 and %q", res.code, res.rep, want)
	}
	if st := ts.get(t, "203.0.113.9", "/api/status"); !st.rep.Paused || st.rep.Error != want {
		t.Errorf("status = %+v, want paused", st.rep)
	}
	if n := ts.cli.count(); n != 3 {
		t.Errorf("command ran %d times, want 3", n)
	}

	// The first conversion stops counting 15 minutes after it started.
	ts.now = ts.now.Add(12 * time.Minute)
	if st := ts.get(t, "203.0.113.9", "/api/status"); st.rep.Paused {
		t.Errorf("still paused after the window: %+v", st.rep)
	}
	if res := ts.post(t, "203.0.113.4", "https://example.com/a", ""); res.code != http.StatusOK {
		t.Errorf("after the window: code = %d, body = %s", res.code, res.body)
	}
}

func TestLimitPerVisitor(t *testing.T) {
	ts := newTestServer(t, Config{Limit: 50, PerVisitor: 2})
	for range 2 {
		if res := ts.post(t, "2001:db8:1:2::1", "https://example.com/a", ""); res.code != http.StatusOK {
			t.Fatalf("code = %d, body = %s", res.code, res.body)
		}
	}
	// Another address in the same IPv6 /64 is the same visitor.
	res := ts.post(t, "2001:db8:1:2::ffff", "https://example.com/a", "")
	if res.code != http.StatusTooManyRequests || !strings.Contains(res.rep.Error, "You've made 2 conversions in the last 15 minutes") {
		t.Errorf("code = %d, reply = %+v", res.code, res.rep)
	}
	if st := ts.get(t, "2001:db8:1:2::1", "/api/status"); !st.rep.Paused {
		t.Errorf("status for the visitor = %+v, want paused", st.rep)
	}
	// Everyone else can still convert.
	if st := ts.get(t, "203.0.113.1", "/api/status"); st.rep.Paused {
		t.Errorf("status for others = %+v, want not paused", st.rep)
	}
	if res := ts.post(t, "2001:db8:1:3::1", "https://example.com/a", ""); res.code != http.StatusOK {
		t.Errorf("other visitor: code = %d, body = %s", res.code, res.body)
	}
}

func TestStorageLimit(t *testing.T) {
	ts := newTestServer(t, Config{MaxStorage: 10})
	if res := ts.post(t, "203.0.113.1", "https://example.com/abcdefgh", ""); res.code != http.StatusOK {
		t.Fatalf("code = %d, body = %s", res.code, res.body)
	}
	if res := ts.post(t, "203.0.113.2", "https://example.com/a", ""); res.code != http.StatusTooManyRequests || !res.rep.Paused {
		t.Errorf("over storage: code = %d, reply = %+v", res.code, res.rep)
	}
	// Once the book expires, conversions resume.
	ts.now = ts.now.Add(25 * time.Hour)
	ageAll(t, ts.cfg.Dir, 25*time.Hour)
	ts.Clean()
	if res := ts.post(t, "203.0.113.2", "https://example.com/a", ""); res.code != http.StatusOK {
		t.Errorf("after clean: code = %d, body = %s", res.code, res.body)
	}
}

func TestBusy(t *testing.T) {
	ts := newTestServer(t, Config{MaxRunning: 1, MaxWaiting: 1, Wait: 50 * time.Millisecond})
	ts.cli.block = make(chan struct{})
	first := make(chan response)
	go func() { first <- ts.post(t, "203.0.113.1", "https://example.com/a", "") }()
	for ts.cli.count() == 0 {
		time.Sleep(time.Millisecond)
	}
	res := ts.post(t, "203.0.113.2", "https://example.com/b", "")
	if res.code != http.StatusServiceUnavailable || !strings.Contains(res.rep.Error, "busy") {
		t.Errorf("code = %d, reply = %+v; want 503 busy", res.code, res.rep)
	}
	close(ts.cli.block)
	if res := <-first; res.code != http.StatusOK {
		t.Errorf("first: code = %d, body = %s", res.code, res.body)
	}
	// The conversion that never ran was taken back.
	ts.Server.mu.Lock()
	n := len(*ts.visitor["203.0.113.2"])
	ts.Server.mu.Unlock()
	if n != 0 {
		t.Errorf("busy visitor has %d conversions counted, want 0", n)
	}
}

func TestFailedConversion(t *testing.T) {
	ts := newTestServer(t, Config{})
	ts.cli.fail = runResult{Code: exitNoContent, Err: "no readable content found"}
	res := ts.post(t, "203.0.113.1", "https://example.com/a", "both")
	if res.code != http.StatusUnprocessableEntity || !strings.Contains(res.rep.Error, "No readable article") {
		t.Errorf("code = %d, reply = %+v", res.code, res.rep)
	}
	if n := ts.cli.count(); n != 1 {
		t.Errorf("command ran %d times, want 1 (no KEPUB after the EPUB failed)", n)
	}
	if entries, _ := os.ReadDir(ts.cfg.Dir); len(entries) != 0 {
		t.Errorf("folder left behind: %v", entries)
	}
}

func TestUnexpectedOutputPath(t *testing.T) {
	ts := newTestServer(t, Config{})
	ts.Server.run = func(context.Context, []string) runResult { return runResult{Path: "/etc/passwd"} }
	res := ts.post(t, "203.0.113.1", "https://example.com/a", "")
	if res.code != http.StatusUnprocessableEntity || res.rep.Files != nil {
		t.Errorf("code = %d, reply = %+v", res.code, res.rep)
	}
}

func TestExplain(t *testing.T) {
	ctx := context.Background()
	tests := []struct {
		res  runResult
		want string
	}{
		{runResult{Code: 1, Err: "couldn't fetch the page: example.com returned 404 Not Found"}, "Couldn't download the page: example.com returned 404 Not Found"},
		{runResult{Code: 1, Err: "couldn't fetch the page: example.com is a local or private address (10.0.0.1); …"}, "local or private network address"},
		{runResult{Code: 2, Err: "-URL 10.0.0.1 is on this computer or a private network; add -LOCAL to convert it"}, "local or private network address"},
		{runResult{Code: 3}, "No readable article"},
		{runResult{Code: 4, Err: "saving: /opt/apps/x: permission denied"}, "Something went wrong"},
		{runResult{Code: -1}, "Something went wrong"},
	}
	for _, tt := range tests {
		if got := explain(ctx, tt.res); !strings.Contains(got, tt.want) {
			t.Errorf("explain(%+v) = %q, want %q", tt.res, got, tt.want)
		}
	}
	expired, cancel := context.WithDeadline(ctx, time.Now().Add(-time.Second))
	defer cancel()
	if got := explain(expired, runResult{Code: -1}); !strings.Contains(got, "too long") {
		t.Errorf("timeout: %q", got)
	}
}

func TestBookLinks(t *testing.T) {
	ts := newTestServer(t, Config{})
	res := ts.post(t, "203.0.113.1", "https://example.com/a", "")
	link := res.rep.Files[0].URL
	if err := os.WriteFile(filepath.Join(filepath.Dir(ts.cfg.Dir), "secret.epub"), []byte("TOP-SECRET-CONTENT"), 0o600); err != nil {
		t.Fatal(err)
	}
	for _, p := range []string{
		"/books/0123456789abcdef0123456789abcdef/a.epub", // unknown id
		strings.TrimSuffix(link, "a.epub") + "b.epub",    // unknown name
		strings.TrimSuffix(link, ".epub") + ".html",      // not a book name
		"/books/../secret.epub",
		"/books/%2e%2e/secret.epub",
		"/books/x/..%2fsecret.epub",
	} {
		if res := ts.get(t, "203.0.113.1", p); res.code == http.StatusOK || strings.Contains(res.body, "TOP-SECRET") {
			t.Errorf("GET %s: code = %d, body = %q", p, res.code, res.body)
		}
	}
	ts.now = ts.now.Add(25 * time.Hour)
	if res := ts.get(t, "203.0.113.1", link); res.code != http.StatusNotFound || !strings.Contains(res.body, "<h1>Link expired</h1>") {
		t.Errorf("expired link: code = %d, body = %q", res.code, res.body)
	}
}

func TestClean(t *testing.T) {
	ts := newTestServer(t, Config{Keep: 2 * time.Hour})
	old := ts.post(t, "203.0.113.1", "https://example.com/old", "").rep.Files[0]
	ageAll(t, ts.cfg.Dir, 3*time.Hour)
	fresh := ts.post(t, "203.0.113.2", "https://example.com/fresh", "").rep.Files[0]
	stray := filepath.Join(ts.cfg.Dir, "not-a-conversion")
	if err := os.Mkdir(stray, 0o750); err != nil {
		t.Fatal(err)
	}
	ts.Clean()
	if _, err := os.Stat(filepath.Join(ts.cfg.Dir, strings.Split(old.URL, "/")[2])); !os.IsNotExist(err) {
		t.Errorf("old conversion kept: %v", err)
	}
	if res := ts.get(t, "203.0.113.1", fresh.URL); res.code != http.StatusOK {
		t.Errorf("fresh book: code = %d", res.code)
	}
	if _, err := os.Stat(stray); err != nil {
		t.Errorf("unrelated folder removed: %v", err)
	}
	if got := ts.stored.Load(); got != fresh.Size {
		t.Errorf("stored = %d, want %d", got, fresh.Size)
	}
	if res := ts.post(t, "203.0.113.1", "https://example.com/x", ""); res.rep.KeepHours != 2 {
		t.Errorf("keepHours = %d, want 2", res.rep.KeepHours)
	}
}

func TestHTMLReply(t *testing.T) {
	ts := newTestServer(t, Config{})
	form := url.Values{"url": {"example.com"}}
	req := httptest.NewRequest("POST", "/api/convert", strings.NewReader(form.Encode()))
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	res := ts.do(t, req)
	if res.code != http.StatusBadRequest || !strings.Contains(res.hdr.Get("Content-Type"), "text/html") {
		t.Fatalf("code = %d, Content-Type = %q", res.code, res.hdr.Get("Content-Type"))
	}

	ts.cli.fail = runResult{Code: 1, Err: `couldn't fetch the page: <script>alert(1)</script>`}
	form = url.Values{"url": {"https://example.com/a"}}
	req = httptest.NewRequest("POST", "/api/convert", strings.NewReader(form.Encode()))
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	res = ts.do(t, req)
	if strings.Contains(res.body, "<script>") || !strings.Contains(res.body, "&lt;script&gt;") {
		t.Errorf("error not escaped:\n%s", res.body)
	}

	ts.cli.fail = runResult{}
	req = httptest.NewRequest("POST", "/api/convert", strings.NewReader(form.Encode()))
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	res = ts.do(t, req)
	if res.code != http.StatusOK || !strings.Contains(res.body, "Links work for 24 hours") || !strings.Contains(res.body, `href="/books/`) {
		t.Errorf("code = %d, body:\n%s", res.code, res.body)
	}
}

func TestCrossOriginRefused(t *testing.T) {
	ts := newTestServer(t, Config{})
	form := url.Values{"url": {"https://example.com/a"}}
	req := httptest.NewRequest("POST", "/api/convert", strings.NewReader(form.Encode()))
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	req.Header.Set("Sec-Fetch-Site", "cross-site")
	if res := ts.do(t, req); res.code != http.StatusForbidden {
		t.Errorf("code = %d, want 403", res.code)
	}
	if n := ts.cli.count(); n != 0 {
		t.Errorf("command ran %d times, want 0", n)
	}
}

func TestVisitorKey(t *testing.T) {
	tests := []struct{ realIP, remote, want string }{
		{"203.0.113.5", "127.0.0.1:5000", "203.0.113.5"},
		{"", "203.0.113.6:5000", "203.0.113.6"},
		{"::ffff:203.0.113.7", "127.0.0.1:5000", "203.0.113.7"},
		{"2001:db8:1:2:3:4:5:6", "127.0.0.1:5000", "2001:db8:1:2::/64"},
	}
	for _, tt := range tests {
		r := httptest.NewRequest("GET", "/", nil)
		r.RemoteAddr = tt.remote
		if tt.realIP != "" {
			r.Header.Set("X-Real-IP", tt.realIP)
		}
		if got := visitorKey(r); got != tt.want {
			t.Errorf("visitorKey(%q, %q) = %q, want %q", tt.realIP, tt.remote, got, tt.want)
		}
	}
}

// TestExecRunner runs a stand-in script through the real runner.
func TestExecRunner(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.Skip("uses a shell script")
	}
	t.Setenv("REKOPUB_TEST_SECRET", "x")
	dir := t.TempDir()
	script := filepath.Join(dir, "rekopub")
	body := "#!/bin/sh\n" +
		`[ -z "$REKOPUB_TEST_SECRET$HOME" ] || { echo "error: environment not empty" >&2; exit 4; }` + "\n" +
		`case "$1" in *fail*) echo "warning: one image" >&2; echo "error: couldn't fetch the page: example.com returned 404 Not Found" >&2; exit 1;; esac` + "\n" +
		`echo "$2" | cut -c7-` + "\n"
	if err := os.WriteFile(script, []byte(body), 0o755); err != nil {
		t.Fatal(err)
	}
	run := execRunner(script)
	if res := run(context.Background(), []string{"-URL=https://example.com/ok", "-SAVE=/x/a.epub"}); res.Code != 0 || res.Path != "/x/a.epub" {
		t.Errorf("ok run = %+v", res)
	}
	res := run(context.Background(), []string{"-URL=https://example.com/fail", "-SAVE=/x/"})
	if res.Code != 1 || res.Err != "couldn't fetch the page: example.com returned 404 Not Found" {
		t.Errorf("failed run = %+v", res)
	}
}

// ageAll makes every file and folder under dir look d older.
func ageAll(t *testing.T, dir string, d time.Duration) {
	t.Helper()
	filepath.Walk(dir, func(p string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		old := info.ModTime().Add(-d)
		return os.Chtimes(p, old, old)
	})
}