ReKoPub/internal/web/server.go

// Package web is the server behind the converter form on
// rekopub.monoexp.com. It checks the address a visitor enters, runs the
// rekopub command on it and serves the finished books for a limited time.
//
// The site is open to everyone, so conversions are limited: in total and
// per visitor within a time window, by how many run at once, and by how much
// disk the kept books may use. nginx serves the rest of the site and passes
// /api/ and /books/ to this server.
package web

import (
	"context"
	"crypto/rand"
	"embed"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"html/template"
	"io/fs"
	"log"
	"net"
	"net/http"
	"net/netip"
	"net/url"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
	"sync/atomic"
	"time"
	"unicode"

	"git.monoexp.com/bajdev/rekopub/internal/fetch"
)

// Config holds the server's settings. New fills in defaults for zero values
// where noted.
type Config struct {
	CLI        string        // path of the rekopub command (required)
	Dir        string        // books are kept here, one folder per conversion (required)
	Keep       time.Duration // how long books are kept (24h)
	Limit      int           // conversions allowed for all visitors together... (50)
	Window     time.Duration // ...and per visitor, within this period (15m)
	PerVisitor int           // conversions allowed per visitor (10)
	MaxRunning int           // conversions running at the same time (2)
	MaxWaiting int           // conversions waiting for a free slot (10)
	Wait       time.Duration // how long a conversion may wait for a slot (60s)
	RunTimeout time.Duration // how long one run of the command may take (2m)
	MaxStorage int64         // bytes of kept books before conversions pause (2 GB)
}

// Formats a visitor can choose, and the command's flags for each book.
var formats = map[string][]struct {
	name  string
	kepub bool
}{
	"epub":  {{"EPUB", false}},
	"kepub": {{"KEPUB", true}},
	"both":  {{"EPUB", false}, {"KEPUB", true}},
}

const (
	maxURLLen  = 2000
	maxBodyLen = 8 << 10
)

var (
	idPattern   = regexp.MustCompile(`^[0-9a-f]{32}$`)
	bookPattern = regexp.MustCompile(`^[a-z0-9]+(-[a-z0-9]+)*(\.kepub)?\.epub$`)
)

// Server handles the converter's requests.
type Server struct {
	cfg     Config
	root    *os.Root
	run     runFunc
	now     func() time.Time
	isLocal func(ctx context.Context, host string) bool

	mu      sync.Mutex
	all     window
	visitor map[string]*window

	slots   chan struct{}
	waiting atomic.Int32
	stored  atomic.Int64 // bytes of kept books
}

// New returns a server for cfg. It creates cfg.Dir if needed.
func New(cfg Config) (*Server, error) {
	if cfg.CLI == "" || cfg.Dir == "" {
		return nil, errors.New("web: CLI and Dir are required")
	}
	def := func(v *time.Duration, d time.Duration) {
		if *v <= 0 {
			*v = d
		}
	}
	defInt := func(v *int, d int) {
		if *v <= 0 {
			*v = d
		}
	}
	def(&cfg.Keep, 24*time.Hour)
	def(&cfg.Window, 15*time.Minute)
	def(&cfg.Wait, time.Minute)
	def(&cfg.RunTimeout, 2*time.Minute)
	defInt(&cfg.Limit, 50)
	defInt(&cfg.PerVisitor, 10)
	defInt(&cfg.MaxRunning, 2)
	defInt(&cfg.MaxWaiting, 10)
	if cfg.MaxStorage <= 0 {
		cfg.MaxStorage = 2 << 30
	}
	// The command prints the absolute path of each book it saves.
	dir, err := filepath.Abs(cfg.Dir)
	if err != nil {
		return nil, err
	}
	cfg.Dir = dir
	if err := os.MkdirAll(cfg.Dir, 0o750); err != nil {
		return nil, err
	}
	root, err := os.OpenRoot(cfg.Dir)
	if err != nil {
		return nil, err
	}
	return &Server{
		cfg:     cfg,
		root:    root,
		run:     execRunner(cfg.CLI),
		now:     time.Now,
		isLocal: fetch.IsLocal,
		visitor: map[string]*window{},
		slots:   make(chan struct{}, cfg.MaxRunning),
	}, nil
}

// Handler returns the server's routes. Cross-origin form posts are refused,
// so other sites can't use the converter through their visitors' browsers.
func (s *Server) Handler() http.Handler {
	mux := http.NewServeMux()
	mux.HandleFunc("GET /api/status", s.status)
	mux.HandleFunc("POST /api/convert", s.convert)
	mux.HandleFunc("GET /books/{id}/{name}", s.book)
	return http.NewCrossOriginProtection().Handler(headers(mux))
}

// headers sets security headers on every response.
func headers(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		h := w.Header()
		h.Set("X-Content-Type-Options", "nosniff")
		h.Set("Referrer-Policy", "no-referrer")
		h.Set("X-Robots-Tag", "noindex")
		h.Set("Content-Security-Policy", "default-src 'none'; style-src 'self'; img-src 'self'; form-action 'self'; base-uri 'none'; frame-ancestors 'none'")
		next.ServeHTTP(w, r)
	})
}

// file is a finished book in a reply.
type file struct {
	Format string `json:"format"`
	Name   string `json:"name"`
	URL    string `json:"url"`
	Size   int64  `json:"size"`
}

// reply is the answer to a status or convert request.
type reply struct {
	Title     string `json:"-"` // heading of the HTML page, if not the default
	Paused    bool   `json:"paused,omitempty"`
	Error     string `json:"error,omitempty"`
	Warning   string `json:"warning,omitempty"`
	Files     []file `json:"files,omitempty"`
	KeepHours int    `json:"keepHours"`
}

func (s *Server) keepHours() int { return int(s.cfg.Keep.Hours()) }

// status tells the page whether the form can be used right now.
func (s *Server) status(w http.ResponseWriter, r *http.Request) {
	rep := reply{KeepHours: s.keepHours()}
	if msg := s.pausedFor(visitorKey(r)); msg != "" {
		rep.Paused, rep.Error = true, msg
	}
	w.Header().Set("Cache-Control", "no-store")
	writeJSON(w, http.StatusOK, rep)
}

// convert checks the form, waits for a free slot and runs the command once
// per chosen format.
func (s *Server) convert(w http.ResponseWriter, r *http.Request) {
	r.Body = http.MaxBytesReader(w, r.Body, maxBodyLen)
	if err := r.ParseForm(); err != nil {
		s.respond(w, r, http.StatusBadRequest, reply{Error: "The form couldn't be read."})
		return
	}
	format := r.PostForm.Get("format")
	if format == "" {
		format = "epub"
	}
	books, ok := formats[format]
	if !ok {
		s.respond(w, r, http.StatusBadRequest, reply{Error: "Choose EPUB, KEPUB or both."})
		return
	}
	u, msg := checkURL(r.PostForm.Get("url"))
	if msg != "" {
		s.respond(w, r, http.StatusBadRequest, reply{Error: msg})
		return
	}
	// The command refuses local addresses anyway (-NOLOCAL); checking here
	// too gives a clear message and doesn't use up a conversion.
	if s.isLocal(r.Context(), u.Hostname()) {
		s.respond(w, r, http.StatusBadRequest, reply{Error: "That address is on a local or private network, which this site doesn't connect to."})
		return
	}

	key := visitorKey(r)
	undo, msg := s.reserve(key)
	if msg != "" {
		log.Printf("limited %s: %s", key, msg)
		s.respond(w, r, http.StatusTooManyRequests, reply{Paused: true, Error: msg})
		return
	}
	release, msg := s.acquire(r.Context())
	if msg != "" {
		undo()
		s.respond(w, r, http.StatusServiceUnavailable, reply{Error: msg})
		return
	}
	defer release()

	start := s.now()
	rep, code := s.convertAll(r.Context(), u, books)
	if rep.Error != "" {
		log.Printf("convert %s %s: failed in %s: %s", u.Hostname(), format, s.now().Sub(start).Round(time.Millisecond), rep.Error)
	} else {
		log.Printf("convert %s %s: ok in %s", u.Hostname(), format, s.now().Sub(start).Round(time.Millisecond))
	}
	s.respond(w, r, code, rep)
}

// convertAll converts u into each of books, in a new folder.
func (s *Server) convertAll(ctx context.Context, u *url.URL, books []struct {
	name  string
	kepub bool
}) (reply, int) {
	rep := reply{KeepHours: s.keepHours()}
	id := newID()
	dir := filepath.Join(s.cfg.Dir, id)
	if err := s.root.Mkdir(id, 0o750); err != nil {
		log.Printf("creating %s: %v", dir, err)
		rep.Error = "Something went wrong while converting that page."
		return rep, http.StatusInternalServerError
	}
	for _, b := range books {
		f, msg := s.runOnce(ctx, u, dir, b.kepub)
		if msg != "" {
			if len(rep.Files) > 0 {
				// The first book is fine; offer it and say what's missing.
				rep.Warning = fmt.Sprintf("The %s couldn't be made: %s", b.name, msg)
				break
			}
			s.root.RemoveAll(id)
			rep.Error = msg
			return rep, http.StatusUnprocessableEntity
		}
		f.Format = b.name
		f.URL = "/books/" + id + "/" + f.Name
		rep.Files = append(rep.Files, f)
		s.stored.Add(f.Size)
	}
	return rep, http.StatusOK
}

// runOnce runs the command for one book and checks what it wrote.
func (s *Server) runOnce(ctx context.Context, u *url.URL, dir string, kepub bool) (file, string) {
	ctx, cancel := context.WithTimeout(ctx, s.cfg.RunTimeout)
	defer cancel()
	// "-FLAG=value" keeps a value from ever being read as a flag.
	args := []string{"-NOLOCAL", "-URL=" + u.String(), "-SAVE=" + dir + string(filepath.Separator)}
	if kepub {
		args = append(args, "-KEPUB")
	}
	res := s.run(ctx, args)
	if res.Code != 0 {
		return file{}, explain(ctx, res)
	}
	name := filepath.Base(res.Path)
	if filepath.Dir(res.Path) != dir || !bookPattern.MatchString(name) {
		log.Printf("unexpected output path %q", res.Path)
		return file{}, "Something went wrong while converting that page."
	}
	info, err := os.Stat(res.Path)
	if err != nil {
		log.Printf("checking output: %v", err)
		return file{}, "Something went wrong while converting that page."
	}
	return file{Name: name, Size: info.Size()}, ""
}

// reserve counts a conversion against the limits, or says why it can't
// start. undo takes the conversion back if it never runs.
func (s *Server) reserve(key string) (undo func(), msg string) {
	s.mu.Lock()
	defer s.mu.Unlock()
	now := s.now()
	if msg := s.pausedLocked(key, now); msg != "" {
		return nil, msg
	}
	v := s.visitor[key]
	if v == nil {
		v = &window{}
		s.visitor[key] = v
	}
	s.all = append(s.all, now)
	*v = append(*v, now)
	return func() {
		s.mu.Lock()
		defer s.mu.Unlock()
		s.all.remove(now)
		v.remove(now)
	}, ""
}

// pausedFor returns why a visitor can't convert right now, or "".
func (s *Server) pausedFor(key string) string {
	s.mu.Lock()
	defer s.mu.Unlock()
	return s.pausedLocked(key, s.now())
}

func (s *Server) pausedLocked(key string, now time.Time) string {
	if s.stored.Load() >= s.cfg.MaxStorage {
		return "Conversions are temporarily paused. Please try again in an hour."
	}
	s.all.prune(now, s.cfg.Window)
	if full, until := s.all.full(now, s.cfg.Limit, s.cfg.Window); full {
		return "Conversions are temporarily paused. Please try again in " + minutes(until.Sub(now)) + "."
	}
	if v := s.visitor[key]; v != nil {
		v.prune(now, s.cfg.Window)
		if full, until := v.full(now, s.cfg.PerVisitor, s.cfg.Window); full {
			return fmt.Sprintf("You've made %d conversions in the last %s. Please try again in %s.",
				s.cfg.PerVisitor, minutes(s.cfg.Window), minutes(until.Sub(now)))
		}
	}
	return ""
}

// acquire waits for a free slot. It gives up when too many conversions are
// already waiting, after cfg.Wait, or when the visitor leaves.
func (s *Server) acquire(ctx context.Context) (release func(), msg string) {
	const busy = "The converter is busy. Please try again in a minute."
	if s.waiting.Add(1) > int32(s.cfg.MaxWaiting) {
		s.waiting.Add(-1)
		return nil, busy
	}
	defer s.waiting.Add(-1)
	timer := time.NewTimer(s.cfg.Wait)
	defer timer.Stop()
	select {
	case s.slots <- struct{}{}:
		return func() { <-s.slots }, ""
	case <-timer.C:
		return nil, busy
	case <-ctx.Done():
		return nil, busy
	}
}

// book serves a finished book as a download.
func (s *Server) book(w http.ResponseWriter, r *http.Request) {
	id, name := r.PathValue("id"), r.PathValue("name")
	gone := func() {
		s.respond(w, r, http.StatusNotFound, reply{Title: "Link expired", Error: fmt.Sprintf(
			"This download link has expired or doesn't exist. Links work for %d hours after a conversion.", s.keepHours())})
	}
	if !idPattern.MatchString(id) || !bookPattern.MatchString(name) {
		gone()
		return
	}
	// os.Root keeps the open inside cfg.Dir, whatever the path holds.
	f, err := s.root.Open(id + "/" + name)
	if err != nil {
		gone()
		return
	}
	defer f.Close()
	info, err := f.Stat()
	if err != nil || !info.Mode().IsRegular() || s.now().Sub(info.ModTime()) >= s.cfg.Keep {
		gone()
		return
	}
	h := w.Header()
	h.Set("Content-Type", "application/epub+zip")
	h.Set("Content-Disposition", `attachment; filename="`+name+`"`)
	h.Set("Cache-Control", "private, max-age=3600")
	http.ServeContent(w, r, name, info.ModTime(), f)
}

// Clean deletes books older than cfg.Keep, adds up what's left and forgets
// visitors with no recent conversions. Run it at start-up and then
// regularly.
func (s *Server) Clean() {
	now := s.now()
	entries, err := fs.ReadDir(s.root.FS(), ".")
	if err != nil {
		log.Printf("clean: %v", err)
		return
	}
	var total int64
	removed := 0
	for _, e := range entries {
		info, err := e.Info()
		if err != nil || !e.IsDir() || !idPattern.MatchString(e.Name()) {
			continue
		}
		if now.Sub(info.ModTime()) >= s.cfg.Keep {
			if err := s.root.RemoveAll(e.Name()); err != nil {
				log.Printf("clean: %v", err)
			} else {
				removed++
			}
			continue
		}
		fs.WalkDir(s.root.FS(), e.Name(), func(_ string, d fs.DirEntry, err error) error {
			if err == nil && d.Type().IsRegular() {
				if fi, err := d.Info(); err == nil {
					total += fi.Size()
				}
			}
			return nil
		})
	}
	s.stored.Store(total)
	if removed > 0 {
		log.Printf("clean: removed %d expired conversions; %d MB kept", removed, total>>20)
	}

	s.mu.Lock()
	defer s.mu.Unlock()
	for key, v := range s.visitor {
		if v.prune(now, s.cfg.Window); len(*v) == 0 {
			delete(s.visitor, key)
		}
	}
}

// CleanEvery runs Clean every interval until ctx ends.
func (s *Server) CleanEvery(ctx context.Context, interval time.Duration) {
	s.Clean()
	t := time.NewTicker(interval)
	defer t.Stop()
	for {
		select {
		case <-ctx.Done():
			return
		case <-t.C:
			s.Clean()
		}
	}
}

// checkURL checks a visitor's address, returning it or a message saying
// what's wrong.
func checkURL(raw string) (*url.URL, string) {
	raw = strings.TrimSpace(raw)
	switch {
	case raw == "":
		return nil, "Enter the address of a web page."
	case len(raw) > maxURLLen:
		return nil, "That address is too long."
	case strings.ContainsFunc(raw, unicode.IsControl):
		return nil, "That isn't a valid web address."
	}
	u, err := url.Parse(raw)
	if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.Opaque != "" {
		return nil, "Enter a full web address starting with http:// or https://."
	}
	if u.User != nil {
		return nil, "Addresses with a user name or password can't be converted."
	}
	// Other ports would let visitors use the server to probe other
	// people's machines.
	if p := u.Port(); p != "" && p != "80" && p != "443" {
		return nil, "Only addresses on the standard web ports can be converted."
	}
	u.Fragment, u.RawFragment = "", ""
	return u, ""
}

// visitorKey identifies a visitor for the per-visitor limit: their IPv4
// address, or the /64 network of an IPv6 address, since one IPv6
// connection usually has a whole /64 to choose from. The server listens on
// 127.0.0.1 behind nginx, which sets X-Real-IP; any value the visitor sent
// is replaced.
func visitorKey(r *http.Request) string {
	raw := r.Header.Get("X-Real-IP")
	if raw == "" {
		raw, _, _ = net.SplitHostPort(r.RemoteAddr)
	}
	ip, err := netip.ParseAddr(raw)
	if err != nil {
		return raw
	}
	ip = ip.Unmap()
	if ip.Is6() {
		p, _ := ip.Prefix(64)
		return p.String()
	}
	return ip.String()
}

func newID() string {
	b := make([]byte, 16)
	rand.Read(b)
	return hex.EncodeToString(b)
}

// respond answers in JSON for the page's script and as a page for a
// browser without JavaScript.
func (s *Server) respond(w http.ResponseWriter, r *http.Request, code int, rep reply) {
	rep.KeepHours = s.keepHours()
	if strings.Contains(r.Header.Get("Accept"), "application/json") {
		writeJSON(w, code, rep)
		return
	}
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	w.WriteHeader(code)
	if err := pages.ExecuteTemplate(w, "result.html", rep); err != nil {
		log.Printf("result page: %v", err)
	}
}

func writeJSON(w http.ResponseWriter, code int, v any) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(code)
	json.NewEncoder(w).Encode(v)
}

//go:embed templates
var templateFS embed.FS

var pages = template.Must(template.New("").Funcs(template.FuncMap{
	"kb": func(n int64) string { return fmt.Sprintf("%d KB", (n+1023)/1024) },
}).ParseFS(templateFS, "templates/*.html"))