package web
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path"
"path/filepath"
"runtime"
"slices"
"strings"
"sync"
"testing"
"time"
)
// fakeCLI stands in for the rekopub command. It writes a book named after
// the page's last path segment into the -SAVE folder, as the command does.
type fakeCLI struct {
mu sync.Mutex
calls [][]string
fail runResult // returned instead of a book when Code != 0
block chan struct{} // if set, runs wait for it to close
}
func (f *fakeCLI) run(ctx context.Context, args []string) runResult {
f.mu.Lock()
f.calls = append(f.calls, args)
f.mu.Unlock()
if f.block != nil {
select {
case <-f.block:
case <-ctx.Done():
return runResult{Code: -1}
}
}
if f.fail.Code != 0 {
return f.fail
}
var page, dir string
kepub := false
for _, a := range args {
switch {
case strings.HasPrefix(a, "-URL="):
u, _ := url.Parse(strings.TrimPrefix(a, "-URL="))
page = filepath.Base(u.Path)
case strings.HasPrefix(a, "-SAVE="):
dir = strings.TrimPrefix(a, "-SAVE=")
case a == "-KEPUB":
kepub = true
}
}
path := filepath.Join(dir, page+".epub")
if kepub {
path = filepath.Join(dir, page+".kepub.epub")
}
if err := os.WriteFile(path, []byte("book:"+page), 0o640); err != nil {
return runResult{Code: 4, Err: err.Error()}
}
abs, _ := filepath.Abs(path) // the command prints absolute paths
return runResult{Path: abs}
}
func (f *fakeCLI) count() int {
f.mu.Lock()
defer f.mu.Unlock()
return len(f.calls)
}
type testServer struct {
*Server
cli *fakeCLI
now time.Time
http http.Handler
}
func newTestServer(t *testing.T, cfg Config) *testServer {
t.Helper()
cfg.CLI = "rekopub"
// Relative, as on the server (DATA_DIR=data).
t.Chdir(t.TempDir())
cfg.Dir = filepath.Join("data", "books")
s, err := New(cfg)
if err != nil {
t.Fatal(err)
}
ts := &testServer{Server: s, cli: &fakeCLI{}, now: time.Now()}
s.run = ts.cli.run
s.now = func() time.Time { return ts.now }
s.isLocal = func(_ context.Context, host string) bool { return host == "localhost" || host == "10.0.0.1" }
ts.http = s.Handler()
return ts
}
type response struct {
code int
body string
rep reply
hdr http.Header
}
func (ts *testServer) do(t *testing.T, req *http.Request) response {
t.Helper()
w := httptest.NewRecorder()
ts.http.ServeHTTP(w, req)
res := response{code: w.Code, body: w.Body.String(), hdr: w.Header()}
if strings.HasPrefix(w.Header().Get("Content-Type"), "application/json") {
if err := json.Unmarshal(w.Body.Bytes(), &res.rep); err != nil {
t.Fatalf("bad JSON %q: %v", res.body, err)
}
}
return res
}
// post submits the form as the page's script does, from visitor ip.
func (ts *testServer) post(t *testing.T, ip, page, format string) response {
t.Helper()
form := url.Values{"url": {page}}
if format != "" {
form.Set("format", format)
}
req := httptest.NewRequest("POST", "/api/convert", strings.NewReader(form.Encode()))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
req.Header.Set("Accept", "application/json")
req.Header.Set("X-Real-IP", ip)
return ts.do(t, req)
}
func (ts *testServer) get(t *testing.T, ip, path string) response {
t.Helper()
req := httptest.NewRequest("GET", path, nil)
req.Header.Set("X-Real-IP", ip)
return ts.do(t, req)
}
func TestConvertEPUB(t *testing.T) {
ts := newTestServer(t, Config{})
res := ts.post(t, "203.0.113.5", "https://example.com/posts/hello#top", "")
if res.code != http.StatusOK || len(res.rep.Files) != 1 {
t.Fatalf("code = %d, body = %s", res.code, res.body)
}
if res.rep.KeepHours != 24 {
t.Errorf("keepHours = %d, want 24", res.rep.KeepHours)
}
f := res.rep.Files[0]
if f.Format != "EPUB" || f.Name != "hello.epub" || f.Size != int64(len("book:hello")) {
t.Errorf("file = %+v", f)
}
args := ts.cli.calls[0]
if args[0] != "-NOLOCAL" || args[1] != "-URL=https://example.com/posts/hello" ||
!strings.HasPrefix(args[2], "-SAVE="+ts.cfg.Dir+string(filepath.Separator)) || !strings.HasSuffix(args[2], string(filepath.Separator)) ||
slices.Contains(args, "-KEPUB") {
t.Errorf("command args = %q", args)
}
dl := ts.get(t, "198.51.100.7", f.URL)
if dl.code != http.StatusOK || dl.body != "book:hello" {
t.Fatalf("download: code = %d, body = %q", dl.code, dl.body)
}
if got := dl.hdr.Get("Content-Disposition"); got != `attachment; filename="hello.epub"` {
t.Errorf("Content-Disposition = %q", got)
}
if got := dl.hdr.Get("Content-Type"); got != "application/epub+zip" {
t.Errorf("Content-Type = %q", got)
}
if got := dl.hdr.Get("X-Content-Type-Options"); got != "nosniff" {
t.Errorf("X-Content-Type-Options = %q", got)
}
}
func TestConvertBoth(t *testing.T) {
ts := newTestServer(t, Config{})
res := ts.post(t, "203.0.113.5", "https://example.com/a", "both")
if res.code != http.StatusOK || len(res.rep.Files) != 2 {
t.Fatalf("code = %d, body = %s", res.code, res.body)
}
if res.rep.Files[0].Name != "a.epub" || res.rep.Files[1].Name != "a.kepub.epub" || res.rep.Files[1].Format != "KEPUB" {
t.Errorf("files = %+v", res.rep.Files)
}
if path.Dir(res.rep.Files[0].URL) != path.Dir(res.rep.Files[1].URL) {
t.Errorf("books of one conversion are in different folders: %+v", res.rep.Files)
}
if !slices.Contains(ts.cli.calls[1], "-KEPUB") {
t.Errorf("second run args = %q, want -KEPUB", ts.cli.calls[1])
}
// Both books count as one conversion.
ts.Server.mu.Lock()
n := len(ts.all)
ts.Server.mu.Unlock()
if n != 1 {
t.Errorf("counted %d conversions, want 1", n)
}
}
func TestConvertKEPUB(t *testing.T) {
ts := newTestServer(t, Config{})
res := ts.post(t, "203.0.113.5", "https://example.com/a", "kepub")
if res.code != http.StatusOK || len(res.rep.Files) != 1 || res.rep.Files[0].Name != "a.kepub.epub" {
t.Fatalf("code = %d, body = %s", res.code, res.body)
}
}
func TestBadRequests(t *testing.T) {
tests := []struct {
name, page, format, want string
}{
{"empty", "", "", "Enter the address"},
{"no scheme", "example.com/a", "", "starting with http"},
{"ftp", "ftp://example.com/a", "", "starting with http"},
{"javascript", "javascript:alert(1)", "", "starting with http"},
{"file", "file:///etc/passwd", "", "starting with http"},
{"no host", "https:///a", "", "starting with http"},
{"credentials", "https://user:pw@example.com/a", "", "user name or password"},
{"port", "https://example.com:3000/a", "", "standard web ports"},
{"control", "https://example.com/a\nb", "", "valid web address"},
{"too long", "https://example.com/" + strings.Repeat("a", maxURLLen), "", "too long"},
{"localhost", "http://localhost/a", "", "local or private"},
{"private", "http://10.0.0.1/a", "", "local or private"},
{"format", "https://example.com/a", "pdf", "Choose EPUB"},
}
ts := newTestServer(t, Config{Limit: 3})
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
res := ts.post(t, "203.0.113.5", tt.page, tt.format)
if res.code != http.StatusBadRequest || !strings.Contains(res.rep.Error, tt.want) {
t.Errorf("code = %d, error = %q; want 400 and %q", res.code, res.rep.Error, tt.want)
}
})
}
if n := ts.cli.count(); n != 0 {
t.Errorf("command ran %d times, want 0", n)
}
// Rejected requests don't use up conversions.
if res := ts.get(t, "203.0.113.5", "/api/status"); res.rep.Paused {
t.Errorf("paused after rejected requests: %+v", res.rep)
}
}
func TestOversizedForm(t *testing.T) {
ts := newTestServer(t, Config{})
res := ts.post(t, "203.0.113.5", "https://example.com/"+strings.Repeat("a", maxBodyLen), "")
if res.code != http.StatusBadRequest {
t.Errorf("code = %d, want 400", res.code)
}
}
func TestLimitForEveryone(t *testing.T) {
ts := newTestServer(t, Config{Limit: 3, PerVisitor: 10, Window: 15 * time.Minute})
for i, ip := range []string{"203.0.113.1", "203.0.113.2", "203.0.113.3"} {
if res := ts.post(t, ip, "https://example.com/a", ""); res.code != http.StatusOK {
t.Fatalf("conversion %d: code = %d, body = %s", i+1, res.code, res.body)
}
ts.now = ts.now.Add(time.Minute)
}
res := ts.post(t, "203.0.113.4", "https://example.com/a", "")
want := "Conversions are temporarily paused. Please try again in 12 minutes."
if res.code != http.StatusTooManyRequests || !res.rep.Paused || res.rep.Error != want {
t.Errorf("code = %d, reply = %+v; want 429 and %q", res.code, res.rep, want)
}
if st := ts.get(t, "203.0.113.9", "/api/status"); !st.rep.Paused || st.rep.Error != want {
t.Errorf("status = %+v, want paused", st.rep)
}
if n := ts.cli.count(); n != 3 {
t.Errorf("command ran %d times, want 3", n)
}
// The first conversion stops counting 15 minutes after it started.
ts.now = ts.now.Add(12 * time.Minute)
if st := ts.get(t, "203.0.113.9", "/api/status"); st.rep.Paused {
t.Errorf("still paused after the window: %+v", st.rep)
}
if res := ts.post(t, "203.0.113.4", "https://example.com/a", ""); res.code != http.StatusOK {
t.Errorf("after the window: code = %d, body = %s", res.code, res.body)
}
}
func TestLimitPerVisitor(t *testing.T) {
ts := newTestServer(t, Config{Limit: 50, PerVisitor: 2})
for range 2 {
if res := ts.post(t, "2001:db8:1:2::1", "https://example.com/a", ""); res.code != http.StatusOK {
t.Fatalf("code = %d, body = %s", res.code, res.body)
}
}
// Another address in the same IPv6 /64 is the same visitor.
res := ts.post(t, "2001:db8:1:2::ffff", "https://example.com/a", "")
if res.code != http.StatusTooManyRequests || !strings.Contains(res.rep.Error, "You've made 2 conversions in the last 15 minutes") {
t.Errorf("code = %d, reply = %+v", res.code, res.rep)
}
if st := ts.get(t, "2001:db8:1:2::1", "/api/status"); !st.rep.Paused {
t.Errorf("status for the visitor = %+v, want paused", st.rep)
}
// Everyone else can still convert.
if st := ts.get(t, "203.0.113.1", "/api/status"); st.rep.Paused {
t.Errorf("status for others = %+v, want not paused", st.rep)
}
if res := ts.post(t, "2001:db8:1:3::1", "https://example.com/a", ""); res.code != http.StatusOK {
t.Errorf("other visitor: code = %d, body = %s", res.code, res.body)
}
}
func TestStorageLimit(t *testing.T) {
ts := newTestServer(t, Config{MaxStorage: 10})
if res := ts.post(t, "203.0.113.1", "https://example.com/abcdefgh", ""); res.code != http.StatusOK {
t.Fatalf("code = %d, body = %s", res.code, res.body)
}
if res := ts.post(t, "203.0.113.2", "https://example.com/a", ""); res.code != http.StatusTooManyRequests || !res.rep.Paused {
t.Errorf("over storage: code = %d, reply = %+v", res.code, res.rep)
}
// Once the book expires, conversions resume.
ts.now = ts.now.Add(25 * time.Hour)
ageAll(t, ts.cfg.Dir, 25*time.Hour)
ts.Clean()
if res := ts.post(t, "203.0.113.2", "https://example.com/a", ""); res.code != http.StatusOK {
t.Errorf("after clean: code = %d, body = %s", res.code, res.body)
}
}
func TestBusy(t *testing.T) {
ts := newTestServer(t, Config{MaxRunning: 1, MaxWaiting: 1, Wait: 50 * time.Millisecond})
ts.cli.block = make(chan struct{})
first := make(chan response)
go func() { first <- ts.post(t, "203.0.113.1", "https://example.com/a", "") }()
for ts.cli.count() == 0 {
time.Sleep(time.Millisecond)
}
res := ts.post(t, "203.0.113.2", "https://example.com/b", "")
if res.code != http.StatusServiceUnavailable || !strings.Contains(res.rep.Error, "busy") {
t.Errorf("code = %d, reply = %+v; want 503 busy", res.code, res.rep)
}
close(ts.cli.block)
if res := <-first; res.code != http.StatusOK {
t.Errorf("first: code = %d, body = %s", res.code, res.body)
}
// The conversion that never ran was taken back.
ts.Server.mu.Lock()
n := len(*ts.visitor["203.0.113.2"])
ts.Server.mu.Unlock()
if n != 0 {
t.Errorf("busy visitor has %d conversions counted, want 0", n)
}
}
func TestFailedConversion(t *testing.T) {
ts := newTestServer(t, Config{})
ts.cli.fail = runResult{Code: exitNoContent, Err: "no readable content found"}
res := ts.post(t, "203.0.113.1", "https://example.com/a", "both")
if res.code != http.StatusUnprocessableEntity || !strings.Contains(res.rep.Error, "No readable article") {
t.Errorf("code = %d, reply = %+v", res.code, res.rep)
}
if n := ts.cli.count(); n != 1 {
t.Errorf("command ran %d times, want 1 (no KEPUB after the EPUB failed)", n)
}
if entries, _ := os.ReadDir(ts.cfg.Dir); len(entries) != 0 {
t.Errorf("folder left behind: %v", entries)
}
}
func TestUnexpectedOutputPath(t *testing.T) {
ts := newTestServer(t, Config{})
ts.Server.run = func(context.Context, []string) runResult { return runResult{Path: "/etc/passwd"} }
res := ts.post(t, "203.0.113.1", "https://example.com/a", "")
if res.code != http.StatusUnprocessableEntity || res.rep.Files != nil {
t.Errorf("code = %d, reply = %+v", res.code, res.rep)
}
}
func TestExplain(t *testing.T) {
ctx := context.Background()
tests := []struct {
res runResult
want string
}{
{runResult{Code: 1, Err: "couldn't fetch the page: example.com returned 404 Not Found"}, "Couldn't download the page: example.com returned 404 Not Found"},
{runResult{Code: 1, Err: "couldn't fetch the page: example.com is a local or private address (10.0.0.1); …"}, "local or private network address"},
{runResult{Code: 3}, "No readable article"},
{runResult{Code: 4, Err: "saving: /opt/apps/x: permission denied"}, "Something went wrong"},
{runResult{Code: -1}, "Something went wrong"},
}
for _, tt := range tests {
if got := explain(ctx, tt.res); !strings.Contains(got, tt.want) {
t.Errorf("explain(%+v) = %q, want %q", tt.res, got, tt.want)
}
}
expired, cancel := context.WithDeadline(ctx, time.Now().Add(-time.Second))
defer cancel()
if got := explain(expired, runResult{Code: -1}); !strings.Contains(got, "too long") {
t.Errorf("timeout: %q", got)
}
}
func TestBookLinks(t *testing.T) {
ts := newTestServer(t, Config{})
res := ts.post(t, "203.0.113.1", "https://example.com/a", "")
link := res.rep.Files[0].URL
if err := os.WriteFile(filepath.Join(filepath.Dir(ts.cfg.Dir), "secret.epub"), []byte("TOP-SECRET-CONTENT"), 0o600); err != nil {
t.Fatal(err)
}
for _, p := range []string{
"/books/0123456789abcdef0123456789abcdef/a.epub", // unknown id
strings.TrimSuffix(link, "a.epub") + "b.epub", // unknown name
strings.TrimSuffix(link, ".epub") + ".html", // not a book name
"/books/../secret.epub",
"/books/%2e%2e/secret.epub",
"/books/x/..%2fsecret.epub",
} {
if res := ts.get(t, "203.0.113.1", p); res.code == http.StatusOK || strings.Contains(res.body, "TOP-SECRET") {
t.Errorf("GET %s: code = %d, body = %q", p, res.code, res.body)
}
}
ts.now = ts.now.Add(25 * time.Hour)
if res := ts.get(t, "203.0.113.1", link); res.code != http.StatusNotFound || !strings.Contains(res.body, "Link expired
") {
t.Errorf("expired link: code = %d, body = %q", res.code, res.body)
}
}
func TestClean(t *testing.T) {
ts := newTestServer(t, Config{Keep: 2 * time.Hour})
old := ts.post(t, "203.0.113.1", "https://example.com/old", "").rep.Files[0]
ageAll(t, ts.cfg.Dir, 3*time.Hour)
fresh := ts.post(t, "203.0.113.2", "https://example.com/fresh", "").rep.Files[0]
stray := filepath.Join(ts.cfg.Dir, "not-a-conversion")
if err := os.Mkdir(stray, 0o750); err != nil {
t.Fatal(err)
}
ts.Clean()
if _, err := os.Stat(filepath.Join(ts.cfg.Dir, strings.Split(old.URL, "/")[2])); !os.IsNotExist(err) {
t.Errorf("old conversion kept: %v", err)
}
if res := ts.get(t, "203.0.113.1", fresh.URL); res.code != http.StatusOK {
t.Errorf("fresh book: code = %d", res.code)
}
if _, err := os.Stat(stray); err != nil {
t.Errorf("unrelated folder removed: %v", err)
}
if got := ts.stored.Load(); got != fresh.Size {
t.Errorf("stored = %d, want %d", got, fresh.Size)
}
if res := ts.post(t, "203.0.113.1", "https://example.com/x", ""); res.rep.KeepHours != 2 {
t.Errorf("keepHours = %d, want 2", res.rep.KeepHours)
}
}
func TestHTMLReply(t *testing.T) {
ts := newTestServer(t, Config{})
form := url.Values{"url": {"example.com"}}
req := httptest.NewRequest("POST", "/api/convert", strings.NewReader(form.Encode()))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
res := ts.do(t, req)
if res.code != http.StatusBadRequest || !strings.Contains(res.hdr.Get("Content-Type"), "text/html") {
t.Fatalf("code = %d, Content-Type = %q", res.code, res.hdr.Get("Content-Type"))
}
ts.cli.fail = runResult{Code: 1, Err: `couldn't fetch the page: `}
form = url.Values{"url": {"https://example.com/a"}}
req = httptest.NewRequest("POST", "/api/convert", strings.NewReader(form.Encode()))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
res = ts.do(t, req)
if strings.Contains(res.body, "